From a7c7632e5450cec2e868795b7da50046a206dc26 Mon Sep 17 00:00:00 2001 From: Stavros Kois Date: Tue, 3 Dec 2024 14:39:07 +0200 Subject: [PATCH 01/19] whoogle: use v2 --- ix-dev/community/whoogle/app.yaml | 6 +- .../whoogle/templates/docker-compose.yaml | 110 +-- .../library/base_v1_1_7/environment.py | 98 --- .../library/base_v1_1_7/healthchecks.py | 120 ---- .../templates/library/base_v1_1_7/mariadb.py | 72 -- .../templates/library/base_v1_1_7/metadata.py | 71 -- .../templates/library/base_v1_1_7/network.py | 21 - .../library/base_v1_1_7/permissions.py | 139 ---- .../templates/library/base_v1_1_7/ports.py | 42 -- .../templates/library/base_v1_1_7/postgres.py | 77 -- .../templates/library/base_v1_1_7/redis.py | 49 -- .../library/base_v1_1_7/resources.py | 101 --- .../templates/library/base_v1_1_7/security.py | 34 - .../templates/library/base_v1_1_7/storage.py | 370 ---------- .../templates/library/base_v1_1_7/utils.py | 124 ---- .../{base_v1_1_7 => base_v2_0_30}/__init__.py | 0 .../library/base_v2_0_30}/configs.py | 0 .../library/base_v2_0_30}/container.py | 0 .../library/base_v2_0_30}/depends.py | 0 .../templates/library/base_v2_0_30}/deploy.py | 0 .../templates/library/base_v2_0_30}/deps.py | 0 .../templates/library/base_v2_0_30}/device.py | 0 .../library/base_v2_0_30}/devices.py | 0 .../templates/library/base_v2_0_30}/dns.py | 0 .../library/base_v2_0_30}/environment.py | 0 .../templates/library/base_v2_0_30}/error.py | 0 .../library/base_v2_0_30}/formatter.py | 0 .../library/base_v2_0_30}/functions.py | 6 + .../library/base_v2_0_30}/healthcheck.py | 0 .../templates/library/base_v2_0_30}/labels.py | 0 .../templates/library/base_v2_0_30}/notes.py | 0 .../templates/library/base_v2_0_30}/portal.py | 0 .../library/base_v2_0_30}/portals.py | 0 .../templates/library/base_v2_0_30}/ports.py | 0 .../templates/library/base_v2_0_30}/render.py | 0 .../library/base_v2_0_30}/resources.py | 0 .../library/base_v2_0_30}/restart.py | 0 .../library/base_v2_0_30}/storage.py | 0 .../library/base_v2_0_30}/sysctls.py | 0 .../library/base_v2_0_30/tests}/__init__.py | 0 .../base_v2_0_30}/tests/test_build_image.py | 0 .../base_v2_0_30}/tests/test_configs.py | 0 .../base_v2_0_30}/tests/test_container.py | 0 .../base_v2_0_30}/tests/test_depends.py | 0 .../library/base_v2_0_30}/tests/test_deps.py | 0 .../base_v2_0_30}/tests/test_device.py | 0 .../library/base_v2_0_30}/tests/test_dns.py | 0 .../base_v2_0_30}/tests/test_environment.py | 0 .../base_v2_0_30}/tests/test_formatter.py | 0 .../base_v2_0_30}/tests/test_functions.py | 6 + .../base_v2_0_30}/tests/test_healthcheck.py | 0 .../base_v2_0_30}/tests/test_labels.py | 0 .../library/base_v2_0_30}/tests/test_notes.py | 0 .../base_v2_0_30}/tests/test_portal.py | 0 .../library/base_v2_0_30}/tests/test_ports.py | 0 .../base_v2_0_30}/tests/test_render.py | 0 .../base_v2_0_30}/tests/test_resources.py | 0 .../base_v2_0_30}/tests/test_restart.py | 0 .../base_v2_0_30}/tests/test_sysctls.py | 0 .../base_v2_0_30}/tests/test_volumes.py | 0 .../library/base_v2_0_30}/validations.py | 0 .../library/base_v2_0_30}/volume_mount.py | 0 .../base_v2_0_30}/volume_mount_types.py | 0 .../library/base_v2_0_30}/volume_sources.py | 0 .../library/base_v2_0_30}/volume_types.py | 0 .../library/base_v2_0_30}/volumes.py | 0 library/{2.0.30/tests => 2.0.31}/__init__.py | 0 library/2.0.31/configs.py | 86 +++ library/2.0.31/container.py | 317 +++++++++ library/2.0.31/depends.py | 34 + library/2.0.31/deploy.py | 24 + library/2.0.31/deps.py | 454 ++++++++++++ library/2.0.31/device.py | 31 + library/2.0.31/devices.py | 66 ++ library/2.0.31/dns.py | 79 +++ library/2.0.31/environment.py | 109 +++ library/2.0.31/error.py | 4 + library/2.0.31/formatter.py | 26 + library/2.0.31/functions.py | 148 ++++ library/2.0.31/healthcheck.py | 193 +++++ library/2.0.31/labels.py | 37 + library/2.0.31/notes.py | 70 ++ library/2.0.31/portal.py | 22 + library/2.0.31/portals.py | 28 + library/2.0.31/ports.py | 68 ++ library/2.0.31/render.py | 89 +++ library/2.0.31/resources.py | 115 +++ library/2.0.31/restart.py | 25 + library/2.0.31/storage.py | 116 +++ library/2.0.31/sysctls.py | 38 + library/2.0.31/tests/__init__.py | 0 library/2.0.31/tests/test_build_image.py | 49 ++ library/2.0.31/tests/test_configs.py | 63 ++ library/2.0.31/tests/test_container.py | 324 +++++++++ library/2.0.31/tests/test_depends.py | 54 ++ library/2.0.31/tests/test_deps.py | 380 ++++++++++ library/2.0.31/tests/test_device.py | 121 ++++ library/2.0.31/tests/test_dns.py | 64 ++ library/2.0.31/tests/test_environment.py | 184 +++++ library/2.0.31/tests/test_formatter.py | 13 + library/2.0.31/tests/test_functions.py | 88 +++ library/2.0.31/tests/test_healthcheck.py | 187 +++++ library/2.0.31/tests/test_labels.py | 88 +++ library/2.0.31/tests/test_notes.py | 213 ++++++ library/2.0.31/tests/test_portal.py | 75 ++ library/2.0.31/tests/test_ports.py | 110 +++ library/2.0.31/tests/test_render.py | 37 + library/2.0.31/tests/test_resources.py | 140 ++++ library/2.0.31/tests/test_restart.py | 57 ++ library/2.0.31/tests/test_sysctls.py | 62 ++ library/2.0.31/tests/test_volumes.py | 666 ++++++++++++++++++ library/2.0.31/validations.py | 227 ++++++ library/2.0.31/volume_mount.py | 92 +++ library/2.0.31/volume_mount_types.py | 72 ++ library/2.0.31/volume_sources.py | 106 +++ library/2.0.31/volume_types.py | 133 ++++ library/2.0.31/volumes.py | 66 ++ library/hashes.yaml | 2 +- 118 files changed, 5899 insertions(+), 1399 deletions(-) delete mode 100644 ix-dev/community/whoogle/templates/library/base_v1_1_7/environment.py delete mode 100644 ix-dev/community/whoogle/templates/library/base_v1_1_7/healthchecks.py delete mode 100644 ix-dev/community/whoogle/templates/library/base_v1_1_7/mariadb.py delete mode 100644 ix-dev/community/whoogle/templates/library/base_v1_1_7/metadata.py delete mode 100644 ix-dev/community/whoogle/templates/library/base_v1_1_7/network.py delete mode 100644 ix-dev/community/whoogle/templates/library/base_v1_1_7/permissions.py delete mode 100644 ix-dev/community/whoogle/templates/library/base_v1_1_7/ports.py delete mode 100644 ix-dev/community/whoogle/templates/library/base_v1_1_7/postgres.py delete mode 100644 ix-dev/community/whoogle/templates/library/base_v1_1_7/redis.py delete mode 100644 ix-dev/community/whoogle/templates/library/base_v1_1_7/resources.py delete mode 100644 ix-dev/community/whoogle/templates/library/base_v1_1_7/security.py delete mode 100644 ix-dev/community/whoogle/templates/library/base_v1_1_7/storage.py delete mode 100644 ix-dev/community/whoogle/templates/library/base_v1_1_7/utils.py rename ix-dev/community/whoogle/templates/library/{base_v1_1_7 => base_v2_0_30}/__init__.py (100%) rename {library/2.0.30 => ix-dev/community/whoogle/templates/library/base_v2_0_30}/configs.py (100%) rename {library/2.0.30 => ix-dev/community/whoogle/templates/library/base_v2_0_30}/container.py (100%) rename {library/2.0.30 => ix-dev/community/whoogle/templates/library/base_v2_0_30}/depends.py (100%) rename {library/2.0.30 => ix-dev/community/whoogle/templates/library/base_v2_0_30}/deploy.py (100%) rename {library/2.0.30 => ix-dev/community/whoogle/templates/library/base_v2_0_30}/deps.py (100%) rename {library/2.0.30 => ix-dev/community/whoogle/templates/library/base_v2_0_30}/device.py (100%) rename {library/2.0.30 => ix-dev/community/whoogle/templates/library/base_v2_0_30}/devices.py (100%) rename {library/2.0.30 => ix-dev/community/whoogle/templates/library/base_v2_0_30}/dns.py (100%) rename {library/2.0.30 => ix-dev/community/whoogle/templates/library/base_v2_0_30}/environment.py (100%) rename {library/2.0.30 => ix-dev/community/whoogle/templates/library/base_v2_0_30}/error.py (100%) rename {library/2.0.30 => ix-dev/community/whoogle/templates/library/base_v2_0_30}/formatter.py (100%) rename {library/2.0.30 => ix-dev/community/whoogle/templates/library/base_v2_0_30}/functions.py (94%) rename {library/2.0.30 => ix-dev/community/whoogle/templates/library/base_v2_0_30}/healthcheck.py (100%) rename {library/2.0.30 => ix-dev/community/whoogle/templates/library/base_v2_0_30}/labels.py (100%) rename {library/2.0.30 => ix-dev/community/whoogle/templates/library/base_v2_0_30}/notes.py (100%) rename {library/2.0.30 => ix-dev/community/whoogle/templates/library/base_v2_0_30}/portal.py (100%) rename {library/2.0.30 => ix-dev/community/whoogle/templates/library/base_v2_0_30}/portals.py (100%) rename {library/2.0.30 => ix-dev/community/whoogle/templates/library/base_v2_0_30}/ports.py (100%) rename {library/2.0.30 => ix-dev/community/whoogle/templates/library/base_v2_0_30}/render.py (100%) rename {library/2.0.30 => ix-dev/community/whoogle/templates/library/base_v2_0_30}/resources.py (100%) rename {library/2.0.30 => ix-dev/community/whoogle/templates/library/base_v2_0_30}/restart.py (100%) rename {library/2.0.30 => ix-dev/community/whoogle/templates/library/base_v2_0_30}/storage.py (100%) rename {library/2.0.30 => ix-dev/community/whoogle/templates/library/base_v2_0_30}/sysctls.py (100%) rename {library/2.0.30 => ix-dev/community/whoogle/templates/library/base_v2_0_30/tests}/__init__.py (100%) rename {library/2.0.30 => ix-dev/community/whoogle/templates/library/base_v2_0_30}/tests/test_build_image.py (100%) rename {library/2.0.30 => ix-dev/community/whoogle/templates/library/base_v2_0_30}/tests/test_configs.py (100%) rename {library/2.0.30 => ix-dev/community/whoogle/templates/library/base_v2_0_30}/tests/test_container.py (100%) rename {library/2.0.30 => ix-dev/community/whoogle/templates/library/base_v2_0_30}/tests/test_depends.py (100%) rename {library/2.0.30 => ix-dev/community/whoogle/templates/library/base_v2_0_30}/tests/test_deps.py (100%) rename {library/2.0.30 => ix-dev/community/whoogle/templates/library/base_v2_0_30}/tests/test_device.py (100%) rename {library/2.0.30 => ix-dev/community/whoogle/templates/library/base_v2_0_30}/tests/test_dns.py (100%) rename {library/2.0.30 => ix-dev/community/whoogle/templates/library/base_v2_0_30}/tests/test_environment.py (100%) rename {library/2.0.30 => ix-dev/community/whoogle/templates/library/base_v2_0_30}/tests/test_formatter.py (100%) rename {library/2.0.30 => ix-dev/community/whoogle/templates/library/base_v2_0_30}/tests/test_functions.py (93%) rename {library/2.0.30 => ix-dev/community/whoogle/templates/library/base_v2_0_30}/tests/test_healthcheck.py (100%) rename {library/2.0.30 => ix-dev/community/whoogle/templates/library/base_v2_0_30}/tests/test_labels.py (100%) rename {library/2.0.30 => ix-dev/community/whoogle/templates/library/base_v2_0_30}/tests/test_notes.py (100%) rename {library/2.0.30 => ix-dev/community/whoogle/templates/library/base_v2_0_30}/tests/test_portal.py (100%) rename {library/2.0.30 => ix-dev/community/whoogle/templates/library/base_v2_0_30}/tests/test_ports.py (100%) rename {library/2.0.30 => ix-dev/community/whoogle/templates/library/base_v2_0_30}/tests/test_render.py (100%) rename {library/2.0.30 => ix-dev/community/whoogle/templates/library/base_v2_0_30}/tests/test_resources.py (100%) rename {library/2.0.30 => ix-dev/community/whoogle/templates/library/base_v2_0_30}/tests/test_restart.py (100%) rename {library/2.0.30 => ix-dev/community/whoogle/templates/library/base_v2_0_30}/tests/test_sysctls.py (100%) rename {library/2.0.30 => ix-dev/community/whoogle/templates/library/base_v2_0_30}/tests/test_volumes.py (100%) rename {library/2.0.30 => ix-dev/community/whoogle/templates/library/base_v2_0_30}/validations.py (100%) rename {library/2.0.30 => ix-dev/community/whoogle/templates/library/base_v2_0_30}/volume_mount.py (100%) rename {library/2.0.30 => ix-dev/community/whoogle/templates/library/base_v2_0_30}/volume_mount_types.py (100%) rename {library/2.0.30 => ix-dev/community/whoogle/templates/library/base_v2_0_30}/volume_sources.py (100%) rename {library/2.0.30 => ix-dev/community/whoogle/templates/library/base_v2_0_30}/volume_types.py (100%) rename {library/2.0.30 => ix-dev/community/whoogle/templates/library/base_v2_0_30}/volumes.py (100%) rename library/{2.0.30/tests => 2.0.31}/__init__.py (100%) create mode 100644 library/2.0.31/configs.py create mode 100644 library/2.0.31/container.py create mode 100644 library/2.0.31/depends.py create mode 100644 library/2.0.31/deploy.py create mode 100644 library/2.0.31/deps.py create mode 100644 library/2.0.31/device.py create mode 100644 library/2.0.31/devices.py create mode 100644 library/2.0.31/dns.py create mode 100644 library/2.0.31/environment.py create mode 100644 library/2.0.31/error.py create mode 100644 library/2.0.31/formatter.py create mode 100644 library/2.0.31/functions.py create mode 100644 library/2.0.31/healthcheck.py create mode 100644 library/2.0.31/labels.py create mode 100644 library/2.0.31/notes.py create mode 100644 library/2.0.31/portal.py create mode 100644 library/2.0.31/portals.py create mode 100644 library/2.0.31/ports.py create mode 100644 library/2.0.31/render.py create mode 100644 library/2.0.31/resources.py create mode 100644 library/2.0.31/restart.py create mode 100644 library/2.0.31/storage.py create mode 100644 library/2.0.31/sysctls.py create mode 100644 library/2.0.31/tests/__init__.py create mode 100644 library/2.0.31/tests/test_build_image.py create mode 100644 library/2.0.31/tests/test_configs.py create mode 100644 library/2.0.31/tests/test_container.py create mode 100644 library/2.0.31/tests/test_depends.py create mode 100644 library/2.0.31/tests/test_deps.py create mode 100644 library/2.0.31/tests/test_device.py create mode 100644 library/2.0.31/tests/test_dns.py create mode 100644 library/2.0.31/tests/test_environment.py create mode 100644 library/2.0.31/tests/test_formatter.py create mode 100644 library/2.0.31/tests/test_functions.py create mode 100644 library/2.0.31/tests/test_healthcheck.py create mode 100644 library/2.0.31/tests/test_labels.py create mode 100644 library/2.0.31/tests/test_notes.py create mode 100644 library/2.0.31/tests/test_portal.py create mode 100644 library/2.0.31/tests/test_ports.py create mode 100644 library/2.0.31/tests/test_render.py create mode 100644 library/2.0.31/tests/test_resources.py create mode 100644 library/2.0.31/tests/test_restart.py create mode 100644 library/2.0.31/tests/test_sysctls.py create mode 100644 library/2.0.31/tests/test_volumes.py create mode 100644 library/2.0.31/validations.py create mode 100644 library/2.0.31/volume_mount.py create mode 100644 library/2.0.31/volume_mount_types.py create mode 100644 library/2.0.31/volume_sources.py create mode 100644 library/2.0.31/volume_types.py create mode 100644 library/2.0.31/volumes.py diff --git a/ix-dev/community/whoogle/app.yaml b/ix-dev/community/whoogle/app.yaml index c4d6c77a8f..a529b2b092 100644 --- a/ix-dev/community/whoogle/app.yaml +++ b/ix-dev/community/whoogle/app.yaml @@ -9,8 +9,8 @@ icon: https://media.sys.truenas.net/apps/whoogle/icons/icon.png keywords: - search - engine -lib_version: 1.1.7 -lib_version_hash: d05e43e25b7dc1736be6cc1efa4b9255368aa346e3e7a4350a38440f29b73186 +lib_version: 2.0.30 +lib_version_hash: 384dc07207a29a9c830b3c8bb65786d0b83a7fd093f0d7f375cdbbcbff397bf2 maintainers: - email: dev@ixsystems.com name: truenas @@ -30,4 +30,4 @@ sources: - https://hub.docker.com/r/benbusby/whoogle-search title: Whoogle train: community -version: 1.0.24 +version: 1.1.0 diff --git a/ix-dev/community/whoogle/templates/docker-compose.yaml b/ix-dev/community/whoogle/templates/docker-compose.yaml index bf79a63c5f..e7b35678c1 100644 --- a/ix-dev/community/whoogle/templates/docker-compose.yaml +++ b/ix-dev/community/whoogle/templates/docker-compose.yaml @@ -1,86 +1,42 @@ -{# Stores storage items that contains info for volumes, vol mounts, perms dirs and perms mounts #} -{% set storage_items = namespace(items=[]) %} -{# Stores the top level volumes #} -{% set volumes = namespace(items={}) %} -{# Stores the container volume mounts #} -{% set volume_mounts = namespace(items=[]) %} -{# Stores the perms container volume mounts #} -{% set perms_mounts = namespace(items=[]) %} -{# Stores the perms container dirs #} -{% set perms_dirs = namespace(items=[]) %} +{% set tpl = ix_lib.base.render.Render(values) %} +{% set c1 = tpl.add_container(values.consts.whoogle_container_name, "image") %} +{% set perm_container = tpl.deps.perms(values.consts.perms_container_name) %} +{% set perms_config = {"uid": values.consts.run_as_user, "gid": values.consts.run_as_group, "mode": "check"} %} -{% do storage_items.items.append(ix_lib.base.storage.storage_item(data={"type": "temporary", "mount_path": "/config"}, - perm_opts={"mount_path": "/mnt/whoogle/config", "mode": "check", "uid": values.consts.run_as_user, "gid": values.consts.run_as_group} -)) %} -{% do storage_items.items.append(ix_lib.base.storage.storage_item(data={"type": "temporary", "mount_path": "/tmp"}, - perm_opts={"mount_path": "/mnt/whoogle/tmp", "mode": "check", "uid": values.consts.run_as_user, "gid": values.consts.run_as_group} -)) %} -{% do storage_items.items.append(ix_lib.base.storage.storage_item(data={"type": "temporary", "mount_path": "/run/tor"}, - perm_opts={"mount_path": "/mnt/whoogle/runtor", "mode": "check", "uid": values.consts.run_as_user, "gid": values.consts.run_as_group} -)) %} -{% do storage_items.items.append(ix_lib.base.storage.storage_item(data={"type": "temporary", "mount_path": "/var/lib/tor"}, - perm_opts={"mount_path": "/mnt/whoogle/varlibtor", "mode": "check", "uid": values.consts.run_as_user, "gid": values.consts.run_as_group} -)) %} +{% do c1.set_user(values.consts.run_as_user, values.consts.run_as_group) %} +{% do c1.healthcheck.set_test("curl", {"port": values.network.web_port, "path": "/healthz"}) %} -{% for store in values.storage.additional_storage %} - {% do storage_items.items.append(ix_lib.base.storage.storage_item(data=store, values=values, - perm_opts={"mount_path": "/mnt/whoogle/dir_%s"|format(loop.index0), "mode": "check", "uid": values.consts.run_as_user, "gid": values.consts.run_as_group} - )) %} +{% set redirects = namespace(items=[]) %} +{% for redirect in values.whoogle.redirects %} + {% do redirects.items.append("%s:%s"|format(redirect.src, redirect.dst)) %} {% endfor %} -{# Add each item to the above lists #} -{% for item in storage_items.items %} - {% if item.vol and volumes.items.update(item.vol) %}{% endif %} - {% if item.vol_mount and volume_mounts.items.append(item.vol_mount) %}{% endif %} - {% if item.perms_item and (perms_dirs.items.append(item.perms_item.perm_dir), perms_mounts.items.append(item.perms_item.vol_mount)) %}{% endif %} -{% endfor %} +{% do c1.environment.add_env("WHOOGLE_REDIRECTS", redirects.items|join(",")) %} +{% do c1.environment.add_env("EXPOSE_PORT", values.network.web_port) %} +{% do c1.environment.add_user_envs(values.whoogle.additional_envs) %} + +{% do c1.ports.add_port(values.network.web_port, values.network.web_port) %} + +{% do c1.add_storage("/config", tpl.funcs.temp_config("temp-config")) %} +{% do perm_container.add_or_skip_action("temp-config", tpl.funcs.temp_config("temp-config"), perms_config) %} + +{% do c1.add_storage("/run/tor", tpl.funcs.temp_config("temp-run-tor")) %} +{% do perm_container.add_or_skip_action("temp-run-tor", tpl.funcs.temp_config("temp-run-tor"), perms_config) %} -{# Containers #} -services: - {{ values.consts.whoogle_container_name }}: - user: {{ "%d:%d" | format(values.consts.run_as_user, values.consts.run_as_group) }} - image: {{ ix_lib.base.utils.get_image(images=values.images, name="image") }} - restart: unless-stopped - deploy: - resources: {{ ix_lib.base.resources.resources(values.resources) | tojson }} - devices: {{ ix_lib.base.resources.get_devices(values.resources) | tojson }} - {% if perms_dirs.items %} - depends_on: - {{ values.consts.perms_container_name }}: - condition: service_completed_successfully - {% endif %} - {% if values.network.host_network %} - network_mode: host - {% endif %} - cap_drop: {{ ix_lib.base.security.get_caps().drop | tojson }} - security_opt: {{ ix_lib.base.security.get_sec_opts() | tojson }} - {% if values.network.dns_opts %} - dns_opt: {{ ix_lib.base.network.dns_opts(values.network.dns_opts) | tojson }} - {% endif %} - {% set test = ix_lib.base.healthchecks.curl_test(port=values.network.web_port, path="/healthz") %} - healthcheck: {{ ix_lib.base.healthchecks.check_health(test) | tojson }} - {% set redirects = namespace(items=[]) %} - {% for redirect in values.whoogle.redirects %} - {% do redirects.items.append("%s:%s"|format(redirect.src, redirect.dst)) %} - {% endfor %} - {% set app_env = { - "EXPOSE_PORT": values.network.web_port, - "WHOOGLE_REDIRECTS": redirects.items|join(","), - } %} - environment: {{ ix_lib.base.environment.envs(app=app_env, user=values.whoogle.additional_envs, values=values) | tojson }} - {% if not values.network.host_network %} - ports: - - {{ ix_lib.base.ports.get_port(port={"target": values.network.web_port, "published": values.network.web_port}) | tojson }} - {% endif %} - volumes: {{ volume_mounts.items | tojson }} - {% if perms_dirs.items %} - {{ values.consts.perms_container_name }}: {{ ix_lib.base.permissions.perms_container(items=perms_dirs.items, volumes=perms_mounts.items) | tojson }} - {% endif %} +{% do c1.add_storage("/var/lib/tor", tpl.funcs.temp_config("temp-var-lib-tor")) %} +{% do perm_container.add_or_skip_action("temp-var-lib-tor", tpl.funcs.temp_config("temp-var-lib-tor"), perms_config) %} -{% if volumes.items %} -volumes: {{ volumes.items | tojson }} +{% for store in values.storage.additional_storage %} + {% do c1.add_storage(store.mount_path, store) %} + {% do perm_container.add_or_skip_action(store.mount_path, store, perms_config) %} +{% endfor %} + +{% if perm_container.has_actions() %} + {% do perm_container.activate() %} + {% do c1.depends.add_dependency(values.consts.perms_container_name, "service_completed_successfully") %} {% endif %} -x-portals: {{ ix_lib.base.metadata.get_portals([{"port": values.network.web_port}]) | tojson }} -x-notes: {{ ix_lib.base.metadata.get_notes("Whoogle") | tojson }} +{% do tpl.portals.add_portal({"port": values.network.web_port}) %} + +{{ tpl.render() | tojson }} diff --git a/ix-dev/community/whoogle/templates/library/base_v1_1_7/environment.py b/ix-dev/community/whoogle/templates/library/base_v1_1_7/environment.py deleted file mode 100644 index be5c8b7347..0000000000 --- a/ix-dev/community/whoogle/templates/library/base_v1_1_7/environment.py +++ /dev/null @@ -1,98 +0,0 @@ -from . import utils -from .resources import get_nvidia_gpus_reservations - - -def envs(app: dict | None = None, user: list | None = None, values: dict | None = None): - app = app or {} - user = user or [] - values = values or {} - result = {} - - if not values: - utils.throw_error("Values cannot be empty in environment.py") - - if not isinstance(user, list): - utils.throw_error( - f"Unsupported type for user environment variables [{type(user)}]" - ) - - # Always set TZ - result.update({"TZ": values.get("TZ", "Etc/UTC")}) - - # Update envs with nvidia variables - if values.get("resources", {}).get("gpus", {}): - result.update(get_nvidia_env(values.get("resources", {}).get("gpus", {}))) - - # Update envs with run_as variables - if values.get("run_as"): - result.update(get_run_as_envs(values.get("run_as", {}))) - - # Make sure we don't manually set any of the above - for item in app.items(): - if not item[0]: - utils.throw_error("Environment variable name cannot be empty.") - if item[0] in result: - utils.throw_error( - f"Environment variable [{item[0]}] is already defined automatically from the library." - ) - result[item[0]] = item[1] - - for item in user: - if not item.get("name"): - utils.throw_error("Environment variable name cannot be empty.") - if item.get("name") in result: - utils.throw_error( - f"Environment variable [{item['name']}] is already defined from the application developer." - ) - result[item["name"]] = item.get("value") - - for k, v in result.items(): - val = str(v) - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - val = val.lower() - result[k] = utils.escape_dollar(val) - - return result - - -# Sets some common variables that most applications use -def get_run_as_envs(run_as: dict) -> dict: - result = {} - user = run_as.get("user") - group = run_as.get("group") - if user: - result.update( - { - "PUID": user, - "UID": user, - "USER_ID": user, - } - ) - if group: - result.update( - { - "PGID": group, - "GID": group, - "GROUP_ID": group, - } - ) - return result - - -def get_nvidia_env(gpus: dict) -> dict: - reservations = get_nvidia_gpus_reservations(gpus) - if not reservations.get("device_ids"): - return { - "NVIDIA_VISIBLE_DEVICES": "void", - } - - return { - "NVIDIA_VISIBLE_DEVICES": ( - ",".join(reservations["device_ids"]) - if reservations.get("device_ids") - else "void" - ), - "NVIDIA_DRIVER_CAPABILITIES": "all", - } diff --git a/ix-dev/community/whoogle/templates/library/base_v1_1_7/healthchecks.py b/ix-dev/community/whoogle/templates/library/base_v1_1_7/healthchecks.py deleted file mode 100644 index cc98270d1d..0000000000 --- a/ix-dev/community/whoogle/templates/library/base_v1_1_7/healthchecks.py +++ /dev/null @@ -1,120 +0,0 @@ -from . import utils - - -def check_health(test, interval=10, timeout=5, retries=30, start_period=10): - if not test: - utils.throw_error("Expected [test] to be set") - - return { - "test": test, - "interval": f"{interval}s", - "timeout": f"{timeout}s", - "retries": retries, - "start_period": f"{start_period}s", - } - - -def mariadb_test(db, config=None): - config = config or {} - if not db: - utils.throw_error("MariaDB container: [db] must be set") - - host = config.get("host", "127.0.0.1") - port = config.get("port", 3306) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$$MARIADB_ROOT_PASSWORD ping" - - -def pg_test(user, db, config=None): - config = config or {} - if not user or not db: - utils.throw_error("Postgres container: [user] and [db] must be set") - - host = config.get("host", "127.0.0.1") - port = config.get("port", 5432) - - return f"pg_isready -h {host} -p {port} -d {db} -U {user}" - - -def redis_test(config=None): - config = config or {} - - host = config.get("host", "127.0.0.1") - port = config.get("port", 6379) - password = "$$REDIS_PASSWORD" - - return f"redis-cli -h {host} -p {port} -a {password} ping | grep -q PONG" - - -def curl_test(port, path, config=None): - config = config or {} - if not port or not path: - utils.throw_error("Expected [port] and [path] to be set") - - scheme = config.get("scheme", "http") - host = config.get("host", "127.0.0.1") - headers = config.get("headers", []) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - utils.throw_error("Expected [header] to be a list of two items") - opts.append(f'--header "{header[0]}: {header[1]}"') - - return f"curl --silent --output /dev/null --show-error --fail {' '.join(opts)} {scheme}://{host}:{port}{path}" - - -def wget_test(port, path, config=None): - config = config or {} - if not port or not path: - utils.throw_error("Expected [port] and [path] to be set") - - scheme = config.get("scheme", "http") - host = config.get("host", "127.0.0.1") - headers = config.get("headers", []) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - utils.throw_error("Expected [header] to be a list of two items") - opts.append(f'--header "{header[0]}: {header[1]}"') - - return f"wget --spider --quiet {' '.join(opts)} {scheme}://{host}:{port}{path}" - - -def http_test(port, path, config=None): - config = config or {} - if not port or not path: - utils.throw_error("Expected [port] and [path] to be set") - - host = config.get("host", "127.0.0.1") - - return f""" - /bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&$${{hc_fd}} && cat <&$${{hc_fd}} | grep "HTTP/1.1 200"' - """ # noqa - - -def netcat_test(port, config=None): - config = config or {} - if not port: - utils.throw_error("Expected [port] to be set") - - host = config.get("host", "127.0.0.1") - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(port, config=None): - config = config or {} - if not port: - utils.throw_error("Expected [port] to be set") - - host = config.get("host", "127.0.0.1") - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" diff --git a/ix-dev/community/whoogle/templates/library/base_v1_1_7/mariadb.py b/ix-dev/community/whoogle/templates/library/base_v1_1_7/mariadb.py deleted file mode 100644 index fc12daf81f..0000000000 --- a/ix-dev/community/whoogle/templates/library/base_v1_1_7/mariadb.py +++ /dev/null @@ -1,72 +0,0 @@ -from . import utils -from .security import get_caps, get_sec_opts -from .network import dns_opts -from .healthchecks import mariadb_test, check_health -from .resources import resources - - -def mariadb_env(user, password, root_password, dbname): - if not user: - utils.throw_error("Expected [user] to be set for mariadb") - if not password: - utils.throw_error("Expected [password] to be set for mariadb") - if not root_password: - utils.throw_error("Expected [root_password] to be set for mariadb") - if not dbname: - utils.throw_error("Expected [dbname] to be set for mariadb") - return { - "MARIADB_USER": user, - "MARIADB_PASSWORD": utils.escape_dollar(password), - "MARIADB_ROOT_PASSWORD": utils.escape_dollar(root_password), - "MARIADB_DATABASE": dbname, - "MARIADB_AUTO_UPGRADE": "true", - } - - -def mariadb_container(data={}): - req_keys = [ - "db_user", - "db_password", - "db_root_password", - "db_name", - "volumes", - "resources", - ] - for key in req_keys: - if not data.get(key): - utils.throw_error(f"Expected [{key}] to be set for mariadb") - - db_user = data["db_user"] - db_password = data["db_password"] - db_root_password = data["db_root_password"] - db_name = data["db_name"] - db_port = data.get("port", 3306) - depends = data.get("depends_on", {}) - depends_on = {} - for key in depends: - depends_on[key] = { - "condition": depends[key].get("condition", "service_completed_successfully") - } - - return { - "image": f"{data.get('image', 'mariadb:10.6')}", - "user": f"{data.get('user', '999')}:{data.get('group', '999')}", - "restart": "unless-stopped", - "cap_drop": get_caps()["drop"], - "security_opt": get_sec_opts(), - **({"dns_opts": dns_opts(data["dns_opts"])} if data.get("dns_opts") else {}), - "healthcheck": check_health(mariadb_test(db=db_name, config={"port": db_port})), - "command": [ - "--port", - str(db_port), - ], - "environment": mariadb_env( - user=db_user, - password=db_password, - root_password=db_root_password, - dbname=db_name, - ), - "volumes": data["volumes"], - "depends_on": depends_on, - "deploy": {"resources": resources(data["resources"])}, - } diff --git a/ix-dev/community/whoogle/templates/library/base_v1_1_7/metadata.py b/ix-dev/community/whoogle/templates/library/base_v1_1_7/metadata.py deleted file mode 100644 index c0a59f8979..0000000000 --- a/ix-dev/community/whoogle/templates/library/base_v1_1_7/metadata.py +++ /dev/null @@ -1,71 +0,0 @@ -from . import utils - - -def get_header(app_name: str): - return f"""# Welcome to TrueNAS SCALE - -Thank you for installing {app_name}! -""" - - -def get_footer(app_name: str): - return f"""## Documentation - -Documentation for {app_name} can be found at https://www.truenas.com/docs. - -## Bug reports - -If you find a bug in this app, please file an issue at -https://ixsystems.atlassian.net or https://github.com/truenas/apps - -## Feature requests or improvements - -If you find a feature request for this app, please file an issue at -https://ixsystems.atlassian.net or https://github.com/truenas/apps -""" - - -def get_notes(app_name: str, body: str = ""): - if not app_name: - utils.throw_error("Expected [app_name] to be set") - - return f"{get_header(app_name)}\n\n{body}\n\n{get_footer(app_name)}" - - -def get_portals(portals: list): - valid_schemes = ["http", "https"] - result = [] - for portal in portals: - # Most apps have a single portal, lets default to a standard name - name = portal.get("name", "Web UI") - scheme = portal.get("scheme", "http") - path = portal.get("path", "/") - - if not name: - utils.throw_error("Expected [portal.name] to be set") - if name in [p["name"] for p in result]: - utils.throw_error( - f"Expected [portal.name] to be unique, got [{', '.join([p['name'] for p in result]+[name])}]" - ) - if scheme not in valid_schemes: - utils.throw_error( - f"Expected [portal.scheme] to be one of [{', '.join(valid_schemes)}], got [{portal['scheme']}]" - ) - if not portal.get("port"): - utils.throw_error("Expected [portal.port] to be set") - if not path.startswith("/"): - utils.throw_error( - f"Expected [portal.path] to start with /, got [{portal['path']}]" - ) - - result.append( - { - "name": name, - "scheme": scheme, - "host": portal.get("host", "0.0.0.0"), - "port": portal["port"], - "path": path, - } - ) - - return result diff --git a/ix-dev/community/whoogle/templates/library/base_v1_1_7/network.py b/ix-dev/community/whoogle/templates/library/base_v1_1_7/network.py deleted file mode 100644 index e4761fd295..0000000000 --- a/ix-dev/community/whoogle/templates/library/base_v1_1_7/network.py +++ /dev/null @@ -1,21 +0,0 @@ -from . import utils - - -def dns_opts(dns_options=None): - dns_options = dns_options or [] - if not dns_options: - return [] - - tracked = {} - disallowed_opts = [] - for opt in dns_options: - key = opt.split(":")[0] - if key in tracked: - utils.throw_error( - f"Expected [dns_opts] to be unique, got [{', '.join([d.split(':')[0] for d in tracked])}]" - ) - if key in disallowed_opts: - utils.throw_error(f"Expected [dns_opts] to not contain [{key}] key.") - tracked[key] = opt - - return dns_options diff --git a/ix-dev/community/whoogle/templates/library/base_v1_1_7/permissions.py b/ix-dev/community/whoogle/templates/library/base_v1_1_7/permissions.py deleted file mode 100644 index 1ce6e60ca1..0000000000 --- a/ix-dev/community/whoogle/templates/library/base_v1_1_7/permissions.py +++ /dev/null @@ -1,139 +0,0 @@ -import jsonschema - -from . import utils - -ITEM_SCHEMA = { - "type": "object", - "properties": { - "dir": {"type": "string"}, - "mode": {"type": "string", "enum": ["always", "check"]}, - "uid": {"type": "integer"}, - "gid": {"type": "integer"}, - "chmod": {"type": "string"}, - "is_temporary": {"type": "boolean"}, - }, - "required": ["dir", "mode", "uid", "gid", "chmod", "is_temporary"], -} - - -def perms_container(items=[], volumes=[]): - if not items: - raise ValueError("Expected [items] to be set for perms_container") - if not volumes: - raise ValueError("Expected [volumes] to be set for perms_container") - - command = [process_dir_shell_func()] - for item in items: - try: - jsonschema.validate(item, ITEM_SCHEMA) - except jsonschema.ValidationError as e: - utils.throw_error(f"Item [{item}] is not valid: {e}") - cmd = [ - "process_dir", - item["dir"], - item["mode"], - str(item["uid"]), - str(item["gid"]), - item["chmod"], - str(item["is_temporary"]).lower(), - ] - command.append(" ".join(cmd)) - - return { - "image": "bash", - "user": "root", - "deploy": { - "resources": { - "limits": {"cpus": "1.0", "memory": "512m"}, - } - }, - "entrypoint": ["bash", "-c"], - "command": ["\n".join(command)], - "volumes": volumes, - } - - -# Don't forget to use double $ for shell variables, -# otherwise docker-compose will try to expand them -def process_dir_shell_func(): - return """ -function process_dir() { - local dir=$$1 - local mode=$$2 - local uid=$$3 - local gid=$$4 - local chmod=$$5 - local is_temporary=$$6 - - local fix_owner="false" - local fix_perms="false" - - if [ -z "$$dir" ]; then - echo "Path is empty, skipping..." - return 0 - fi - - if [ ! -d "$$dir" ]; then - echo "Path [$$dir] does is not a directory, skipping..." - return 0 - fi - - if [ "$$is_temporary" = "true" ]; then - echo "Path [$$dir] is a temporary directory, ensuring it is empty..." - # Exclude the safe directory, where we can use to mount files temporarily - find "$$dir" -mindepth 1 -maxdepth 1 ! -name "ix-safe" -exec rm -rf {} + - fi - - if [ "$$is_temporary" = "false" ] && [ -n "$$(ls -A $$dir)" ]; then - echo "Path [$$dir] is not empty, skipping..." - return 0 - fi - - echo "Current Ownership and Permissions on [$$dir]:" - echo "chown: $$(stat -c "%u %g" "$$dir")" - echo "chmod: $$(stat -c "%a" "$$dir")" - - if [ "$$mode" = "always" ]; then - fix_owner="true" - fix_perms="true" - fi - - if [ "$$mode" = "check" ]; then - if [ $$(stat -c %u "$$dir") -eq $$uid ] && [ $$(stat -c %g "$$dir") -eq $$gid ]; then - echo "Ownership is correct. Skipping..." - fix_owner="false" - else - echo "Ownership is incorrect. Fixing..." - fix_owner="true" - fi - - if [ "$$chmod" = "false" ]; then - echo "Skipping permissions check, chmod is false" - elif [ -n "$$chmod" ]; then - if [ $$(stat -c %a "$$dir") -eq $$chmod ]; then - echo "Permissions are correct. Skipping..." - fix_perms="false" - else - echo "Permissions are incorrect. Fixing..." - fix_perms="true" - fi - fi - fi - - if [ "$$fix_owner" = "true" ]; then - echo "Changing ownership to $$uid:$$gid on: [$$dir]" - chown -R "$$uid:$$gid" "$$dir" - echo "Finished changing ownership" - echo "Ownership after changes:" - stat -c "%u %g" "$$dir" - fi - - if [ -n "$$chmod" ] && [ "$$fix_perms" = "true" ]; then - echo "Changing permissions to $$chmod on: [$$dir]" - chmod -R "$$chmod" "$$dir" - echo "Finished changing permissions" - echo "Permissions after changes:" - stat -c "%a" "$$dir" - fi -} -""" diff --git a/ix-dev/community/whoogle/templates/library/base_v1_1_7/ports.py b/ix-dev/community/whoogle/templates/library/base_v1_1_7/ports.py deleted file mode 100644 index c895b47a44..0000000000 --- a/ix-dev/community/whoogle/templates/library/base_v1_1_7/ports.py +++ /dev/null @@ -1,42 +0,0 @@ -import ipaddress - -from . import utils - - -def must_valid_port(num: int): - if num < 1 or num > 65535: - utils.throw_error(f"Expected a valid port number, got [{num}]") - - -def must_valid_ip(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - utils.throw_error(f"Expected a valid IP address, got [{ip}]") - - -def must_valid_protocol(protocol: str): - if protocol not in ["tcp", "udp"]: - utils.throw_error(f"Expected a valid protocol, got [{protocol}]") - - -def must_valid_mode(mode: str): - if mode not in ["ingress", "host"]: - utils.throw_error(f"Expected a valid mode, got [{mode}]") - - -def get_port(port=None): - port = port or {} - must_valid_port(port["published"]) - must_valid_port(port["target"]) - must_valid_ip(port.get("host_ip", "0.0.0.0")) - must_valid_protocol(port.get("protocol", "tcp")) - must_valid_mode(port.get("mode", "ingress")) - - return { - "target": port["target"], - "published": port["published"], - "protocol": port.get("protocol", "tcp"), - "mode": port.get("mode", "ingress"), - "host_ip": port.get("host_ip", "0.0.0.0"), - } diff --git a/ix-dev/community/whoogle/templates/library/base_v1_1_7/postgres.py b/ix-dev/community/whoogle/templates/library/base_v1_1_7/postgres.py deleted file mode 100644 index c5f8275454..0000000000 --- a/ix-dev/community/whoogle/templates/library/base_v1_1_7/postgres.py +++ /dev/null @@ -1,77 +0,0 @@ -from . import utils -from .security import get_caps, get_sec_opts -from .network import dns_opts -from .healthchecks import pg_test, check_health -from .resources import resources - - -def pg_url(variant, host, user, password, dbname, port=5432): - if not host: - utils.throw_error("Expected [host] to be set") - if not user: - utils.throw_error("Expected [user] to be set") - if not password: - utils.throw_error("Expected [password] to be set") - if not dbname: - utils.throw_error("Expected [dbname] to be set") - - if variant == "postgresql": - return f"postgresql://{user}:{password}@{host}:{port}/{dbname}?sslmode=disable" - elif variant == "postgres": - return f"postgres://{user}:{password}@{host}:{port}/{dbname}?sslmode=disable" - else: - utils.throw_error( - f"Expected [variant] to be one of [postgresql, postgres], got [{variant}]" - ) - - -def pg_env(user, password, dbname, port=5432): - if not user: - utils.throw_error("Expected [user] to be set for postgres") - if not password: - utils.throw_error("Expected [password] to be set for postgres") - if not dbname: - utils.throw_error("Expected [dbname] to be set for postgres") - return { - "POSTGRES_USER": user, - "POSTGRES_PASSWORD": utils.escape_dollar(password), - "POSTGRES_DB": dbname, - "POSTGRES_PORT": port, - } - - -def pg_container(data={}): - req_keys = ["db_user", "db_password", "db_name", "volumes", "resources"] - for key in req_keys: - if not data.get(key): - utils.throw_error(f"Expected [{key}] to be set for postgres") - - pg_user = data["db_user"] - pg_password = data["db_password"] - pg_dbname = data["db_name"] - pg_port = data.get("port", 5432) - depends = data.get("depends_on", {}) - depends_on = {} - for key in depends: - depends_on[key] = { - "condition": depends[key].get("condition", "service_completed_successfully") - } - - return { - "image": f"{data.get('image', 'postgres:15')}", - "user": f"{data.get('user', '999')}:{data.get('group', '999')}", - "restart": "unless-stopped", - "cap_drop": get_caps()["drop"], - "security_opt": get_sec_opts(), - **({"dns_opts": dns_opts(data["dns_opts"])} if data.get("dns_opts") else {}), - "healthcheck": check_health(pg_test(user=pg_user, db=pg_dbname)), - "environment": pg_env( - user=pg_user, - password=pg_password, - dbname=pg_dbname, - port=pg_port, - ), - "volumes": data["volumes"], - "depends_on": depends_on, - "deploy": {"resources": resources(data["resources"])}, - } diff --git a/ix-dev/community/whoogle/templates/library/base_v1_1_7/redis.py b/ix-dev/community/whoogle/templates/library/base_v1_1_7/redis.py deleted file mode 100644 index 2356d0e06c..0000000000 --- a/ix-dev/community/whoogle/templates/library/base_v1_1_7/redis.py +++ /dev/null @@ -1,49 +0,0 @@ -from . import utils -from .security import get_caps, get_sec_opts -from .network import dns_opts -from .healthchecks import redis_test, check_health -from .resources import resources - - -def redis_container(data={}): - req_keys = ["password", "volumes", "resources"] - for key in req_keys: - if not data.get(key): - utils.throw_error(f"Expected [{key}] to be set for postgres") - - redis_password = data["password"] - redis_port = data.get("port", 6379) - depends = data.get("depends_on", {}) - depends_on = {} - for key in depends: - depends_on[key] = { - "condition": depends[key].get("condition", "service_completed_successfully") - } - - return { - "image": f"{data.get('image', 'bitnami/redis:7.0.11')}", - "user": f"{data.get('user', '1001')}:{data.get('group', '0')}", - "restart": "unless-stopped", - "cap_drop": get_caps()["drop"], - "security_opt": get_sec_opts(), - **({"dns_opts": dns_opts(data["dns_opts"])} if data.get("dns_opts") else {}), - "healthcheck": check_health(redis_test(config={"port": redis_port})), - "environment": redis_env( - password=redis_password, - port=redis_port, - ), - "volumes": data["volumes"], - "depends_on": depends_on, - "deploy": {"resources": resources(data["resources"])}, - } - - -def redis_env(password, port=6379): - if not password: - utils.throw_error("Expected [password] to be set for redis") - - return { - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": utils.escape_dollar(password), - "REDIS_PORT_NUMBER": port, - } diff --git a/ix-dev/community/whoogle/templates/library/base_v1_1_7/resources.py b/ix-dev/community/whoogle/templates/library/base_v1_1_7/resources.py deleted file mode 100644 index d3235fd565..0000000000 --- a/ix-dev/community/whoogle/templates/library/base_v1_1_7/resources.py +++ /dev/null @@ -1,101 +0,0 @@ -import re - -from . import utils - - -def resources(resources, disable_resource_limits=False): - gpus = resources.get("gpus", {}) - cpus = str(resources.get("limits", {}).get("cpus", 2.0)) - memory = str(resources.get("limits", {}).get("memory", 4096)) - if not re.match(r"^[1-9][0-9]*(\.[0-9]+)?$", cpus): - utils.throw_error(f"Expected cpus to be a number or a float, got [{cpus}]") - if not re.match(r"^[1-9][0-9]*$", memory): - raise ValueError(f"Expected memory to be a number, got [{memory}]") - - result = { - "limits": {"cpus": cpus, "memory": f"{memory}M"}, - "reservations": {"devices": []}, - } - - if gpus: - gpu_result = get_nvidia_gpus_reservations(gpus) - if gpu_result: - # Appending to devices, as we can later extend this to support other types of devices. Eg. TPUs. - result["reservations"]["devices"].append(get_nvidia_gpus_reservations(gpus)) - - # Docker does not like empty "things" all around. - if not result["reservations"]["devices"]: - del result["reservations"] - - if disable_resource_limits: - del result["limits"] - - return result - - -def get_nvidia_gpus_reservations(gpus: dict) -> dict: - """ - Input: - { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - """ - if not gpus: - return {} - - device_ids = [] - for pci, gpu in gpus.get("nvidia_gpu_selection", {}).items(): - if gpu["use_gpu"]: - if not gpu.get("uuid"): - utils.throw_error( - "Expected [uuid] to be set for GPU in" - f"slot [{pci}] in [nvidia_gpu_selection]" - ) - device_ids.append(gpu["uuid"]) - - if not device_ids: - return {} - - return { - "capabilities": ["gpu"], - "driver": "nvidia", - "device_ids": device_ids, - } - - -disallowed_devices = ["/dev/dri"] - - -# Returns the top level devices list -# Accepting other_devices to allow manually adding devices -# directly to the list. (Eg sound devices) -def get_devices(resources: dict, other_devices: list = []) -> list: - devices = [] - if resources.get("gpus", {}).get("use_all_gpus", False): - devices.append("/dev/dri:/dev/dri") - - added_host_devices: list = [] - for device in other_devices: - host_device = device.get("host_device", "").rstrip("/") - container_device = device.get("container_device", "") or host_device - if not host_device: - utils.throw_error(f"Expected [host_device] to be set for device [{device}]") - if not utils.valid_path(host_device): - utils.throw_error( - f"Expected [host_device] to be a valid path for device [{device}]" - ) - if host_device in disallowed_devices: - utils.throw_error( - f"Device [{host_device}] is not allowed to be manually added." - ) - if host_device in added_host_devices: - utils.throw_error( - f"Expected devices to be unique, but [{host_device}] was already added." - ) - devices.append(f"{host_device}:{container_device}") - added_host_devices.append(host_device) - - return devices diff --git a/ix-dev/community/whoogle/templates/library/base_v1_1_7/security.py b/ix-dev/community/whoogle/templates/library/base_v1_1_7/security.py deleted file mode 100644 index b67668b51b..0000000000 --- a/ix-dev/community/whoogle/templates/library/base_v1_1_7/security.py +++ /dev/null @@ -1,34 +0,0 @@ -from base64 import b64encode - -from . import utils - - -def get_caps(add=None, drop=None): - add = add or [] - drop = drop or ["ALL"] - result = {"drop": drop} - if add: - result["add"] = add - return result - - -def get_sec_opts(add=None, remove=None): - add = add or [] - remove = remove or [] - result = ["no-new-privileges"] - for opt in add: - if opt not in result: - result.append(opt) - for opt in remove: - if opt in result: - result.remove(opt) - return result - - -def htpasswd(username, password): - hashed = utils.bcrypt_hash(password) - return username + ":" + hashed - - -def basic_auth(username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") diff --git a/ix-dev/community/whoogle/templates/library/base_v1_1_7/storage.py b/ix-dev/community/whoogle/templates/library/base_v1_1_7/storage.py deleted file mode 100644 index de09ba0c3e..0000000000 --- a/ix-dev/community/whoogle/templates/library/base_v1_1_7/storage.py +++ /dev/null @@ -1,370 +0,0 @@ -import re -import json -import hashlib - -from . import utils - - -BIND_TYPES = ["host_path", "ix_volume"] -VOL_TYPES = ["volume", "nfs", "cifs", "temporary"] -ALL_TYPES = BIND_TYPES + VOL_TYPES + ["tmpfs", "anonymous"] -PROPAGATION_TYPES = ["shared", "slave", "private", "rshared", "rslave", "rprivate"] - - -def _get_name_for_temporary(data): - if not data.get("mount_path"): - utils.throw_error("Expected [mount_path] to be set for temporary volume") - - return data["mount_path"].lstrip("/").lower().replace("/", "_").replace(".", "_").replace(" ", "_") - - -# Returns a volume mount object (Used in container's "volumes" level) -def vol_mount(data, values=None): - values = values or {} - ix_volumes = values.get("ix_volumes") or [] - vol_type = _get_docker_vol_type(data) - - volume = { - "type": vol_type, - "target": utils.valid_path(data.get("mount_path", "")), - "read_only": data.get("read_only", False), - } - if vol_type == "bind": # Default create_host_path is true in short-syntax - volume.update(_get_bind_vol_config(data, values, ix_volumes)) - elif vol_type == "volume": - volume.update(_get_volume_vol_config(data)) - elif vol_type == "tmpfs": - volume.update(_get_tmpfs_vol_config(data)) - elif vol_type == "temporary": - volume["type"] = "volume" - volume.update(_get_volume_vol_config(data)) - elif vol_type == "anonymous": - volume["type"] = "volume" - volume.update(_get_anonymous_vol_config(data)) - - return volume - - -def storage_item(data, values=None, perm_opts=None): - values = values or {} - perm_opts = perm_opts or {} - if data.get("type") == "temporary": - data.update({"volume_name": _get_name_for_temporary(data)}) - return { - "vol_mount": vol_mount(data, values), - "vol": vol(data), - "perms_item": perms_item(data, values, perm_opts) if perm_opts else {}, - } - - -def perms_item(data, values=None, opts=None): - opts = opts or {} - values = values or {} - vol_type = data.get("type", "") - - # Temp volumes are always auto permissions - if vol_type == "temporary": - data.update({"auto_permissions": True}) - - # If its ix_volume, we need to set auto permissions - if vol_type == "ix_volume": - data.update({"auto_permissions": True}) - - if not data.get("auto_permissions"): - return {} - - if vol_type == "host_path": - if data.get("host_path_config", {}).get("acl_enable", False): - return {} - if vol_type == "ix_volume": - if data.get("ix_volume_config", {}).get("acl_enable", False): - return {} - - req_keys = ["mount_path", "mode", "uid", "gid"] - for key in req_keys: - if opts.get(key, None) is None: - utils.throw_error(f"Expected opts passed to [perms_item] to have [{key}] key") - - data.update({"mount_path": opts["mount_path"]}) - volume_mount = vol_mount(data, values) - # For perms volume mount, always set read_only to false - volume_mount.update({"read_only": False}) - - return { - "vol_mount": volume_mount, - "perm_dir": { - "dir": volume_mount["target"], - "mode": opts["mode"], - "uid": opts["uid"], - "gid": opts["gid"], - "chmod": opts.get("chmod", "false"), - "is_temporary": data["type"] == "temporary", - }, - } - - -def create_host_path_default(values): - """ - By default, do not create host path for bind mounts if it does not exist. - If the ix_context is missing, we are either in local dev or CI. - We should create the host path by default there to ease development. - The _magic_ "dev_mode" flag is added so we can also toggle this behavior - in CI, while we are also using ix_context for other tests. - """ - ix_ctx = values.get("ix_context", {}) - if not ix_ctx: - return True - if "dev_mode" in ix_ctx: - return ix_ctx["dev_mode"] - return False - - -def _get_bind_vol_config(data, values, ix_volumes=None): - ix_volumes = ix_volumes or [] - path = host_path(data, ix_volumes) - if data.get("propagation", "rprivate") not in PROPAGATION_TYPES: - utils.throw_error( - f"Expected [propagation] to be one of [{', '.join(PROPAGATION_TYPES)}], got [{data['propagation']}]" - ) - - # https://docs.docker.com/storage/bind-mounts/#configure-bind-propagation - return { - "source": path, - "bind": { - "create_host_path": data.get("host_path_config", {}).get( - "create_host_path", create_host_path_default(values) - ), - "propagation": _get_valid_propagation(data), - }, - } - - -def _get_volume_vol_config(data): - if data.get("type") in ["nfs", "cifs"]: - if data.get("volume_name"): - utils.throw_error("Expected [volume_name] to be empty for [nfs, cifs] type") - data.update({"volume_name": _get_name_for_external_volume(data)}) - if not data.get("volume_name"): - utils.throw_error("Expected [volume_name] to be set for [volume] type") - - return {"source": data["volume_name"], "volume": _process_volume_config(data)} - - -def _get_anonymous_vol_config(data): - return {"volume": _process_volume_config(data)} - - -mode_regex = re.compile(r"^0[0-7]{3}$") - - -def _get_tmpfs_vol_config(data): - tmpfs = {} - config = data.get("tmpfs_config", {}) - - if config.get("size"): - if not isinstance(config["size"], int): - utils.throw_error("Expected [size] to be an integer for [tmpfs] type") - if not config["size"] > 0: - utils.throw_error("Expected [size] to be greater than 0 for [tmpfs] type") - # Convert Mebibytes to Bytes - tmpfs.update({"size": config["size"] * 1024 * 1024}) - - if config.get("mode"): - if not mode_regex.match(str(config["mode"])): - utils.throw_error(f"Expected [mode] to be a octal string for [tmpfs] type, got [{config['mode']}]") - tmpfs.update({"mode": int(config["mode"], 8)}) - - return {"tmpfs": tmpfs} - - -# We generate a unique name for the volume based on the config -# Docker will not update any volume after creation. This is to ensure -# that changing any value (eg server address) in the config will result in a new volume -def _get_name_for_external_volume(data): - config_hash = hashlib.sha256(json.dumps(data).encode("utf-8")).hexdigest() - return f"{data['type']}_{config_hash}" - - -# Returns a volume object (Used in top "volumes" level) -def vol(data): - if not data or _get_docker_vol_type(data) != "volume": - return {} - - if not data.get("volume_name"): - utils.throw_error("Expected [volume_name] to be set for [volume] type") - - if data["type"] == "nfs": - return {data["volume_name"]: _process_nfs(data)} - elif data["type"] == "cifs": - return {data["volume_name"]: _process_cifs(data)} - else: - return {data["volume_name"]: {}} - - -def _is_host_path(data): - return data.get("type") == "host_path" - - -def _get_valid_propagation(data): - if not data.get("propagation"): - return "rprivate" - if not data["propagation"] in PROPAGATION_TYPES: - utils.throw_error( - f"Expected [propagation] to be one of [{', '.join(PROPAGATION_TYPES)}], got [{data['propagation']}]" - ) - return data["propagation"] - - -def _is_ix_volume(data): - return data.get("type") == "ix_volume" - - -# Returns the host path for a for either a host_path or ix_volume -def host_path(data, ix_volumes=None): - ix_volumes = ix_volumes or [] - path = "" - if _is_host_path(data): - path = _process_host_path_config(data) - elif _is_ix_volume(data): - path = _process_ix_volume_config(data, ix_volumes) - else: - utils.throw_error( - f"Expected [host_path()] to be called only for types [host_path, ix_volume], got [{data['type']}]" - ) - - return utils.valid_path(path) - - -# Returns the type of storage as used in docker-compose -def _get_docker_vol_type(data): - if not data.get("type"): - utils.throw_error("Expected [type] to be set for storage") - - if data["type"] not in ALL_TYPES: - utils.throw_error(f"Expected storage [type] to be one of {ALL_TYPES}, got [{data['type']}]") - - if data["type"] in BIND_TYPES: - return "bind" - elif data["type"] in VOL_TYPES: - return "volume" - else: - return data["type"] - - -def _process_host_path_config(data): - if data.get("host_path_config", {}).get("acl_enable", False): - if not data["host_path_config"].get("acl", {}).get("path"): - utils.throw_error("Expected [host_path_config.acl.path] to be set for [host_path] type with ACL enabled") - return data["host_path_config"]["acl"]["path"] - - if not data.get("host_path_config", {}).get("path"): - utils.throw_error("Expected [host_path_config.path] to be set for [host_path] type") - - return data["host_path_config"]["path"] - - -def _process_volume_config(data): - return {"nocopy": data.get("volume_config", {}).get("nocopy", False)} - - -def _process_ix_volume_config(data, ix_volumes): - path = "" - if not data.get("ix_volume_config", {}).get("dataset_name"): - utils.throw_error("Expected [ix_volume_config.dataset_name] to be set for [ix_volume] type") - - if not ix_volumes: - utils.throw_error("Expected [ix_volumes] to be set for [ix_volume] type") - - ds = data["ix_volume_config"]["dataset_name"] - path = ix_volumes.get(ds, None) - if not path: - utils.throw_error(f"Expected the key [{ds}] to be set in [ix_volumes]") - - return path - - -# Constructs a volume object for a cifs type -def _process_cifs(data): - if not data.get("cifs_config"): - utils.throw_error("Expected [cifs_config] to be set for [cifs] type") - - required_keys = ["server", "path", "username", "password"] - for key in required_keys: - if not data["cifs_config"].get(key): - utils.throw_error(f"Expected [{key}] to be set for [cifs] type") - - opts = [ - "noperm", - f"user={data['cifs_config']['username']}", - f"password={data['cifs_config']['password']}", - ] - if data["cifs_config"].get("domain"): - opts.append(f'domain={data["cifs_config"]["domain"]}') - - if data["cifs_config"].get("options"): - if not isinstance(data["cifs_config"]["options"], list): - utils.throw_error("Expected [cifs_config.options] to be a list for [cifs] type") - - disallowed_opts = ["user", "password", "domain", "noperm"] - for opt in data["cifs_config"]["options"]: - if not isinstance(opt, str): - utils.throw_error("Expected [cifs_config.options] to be a list of strings for [cifs] type") - - key = opt.split("=")[0] - for disallowed in disallowed_opts: - if key == disallowed: - utils.throw_error( - f"Expected [cifs_config.options] to not start with [{disallowed}] for [cifs] type" - ) - - opts.append(opt) - - server = data["cifs_config"]["server"].lstrip("/") - path = data["cifs_config"]["path"].strip("/") - volume = { - "driver_opts": { - "type": "cifs", - "device": f"//{server}/{path}", - "o": f"{','.join([utils.escape_dollar(opt) for opt in opts])}", - }, - } - - return volume - - -# Constructs a volume object for a nfs type -def _process_nfs(data): - if not data.get("nfs_config"): - utils.throw_error("Expected [nfs_config] to be set for [nfs] type") - - required_keys = ["server", "path"] - for key in required_keys: - if not data["nfs_config"].get(key): - utils.throw_error(f"Expected [{key}] to be set for [nfs] type") - - opts = [f"addr={data['nfs_config']['server']}"] - if data["nfs_config"].get("options"): - if not isinstance(data["nfs_config"]["options"], list): - utils.throw_error("Expected [nfs_config.options] to be a list for [nfs] type") - - disallowed_opts = ["addr"] - for opt in data["nfs_config"]["options"]: - if not isinstance(opt, str): - utils.throw_error("Expected [nfs_config.options] to be a list of strings for [nfs] type") - - key = opt.split("=")[0] - for disallowed in disallowed_opts: - if key == disallowed: - utils.throw_error(f"Expected [nfs_config.options] to not start with [{disallowed}] for [nfs] type") - - opts.append(opt) - - volume = { - "driver_opts": { - "type": "nfs", - "device": f":{data['nfs_config']['path']}", - "o": f"{','.join([utils.escape_dollar(opt) for opt in opts])}", - }, - } - - return volume diff --git a/ix-dev/community/whoogle/templates/library/base_v1_1_7/utils.py b/ix-dev/community/whoogle/templates/library/base_v1_1_7/utils.py deleted file mode 100644 index 8a7c0815c6..0000000000 --- a/ix-dev/community/whoogle/templates/library/base_v1_1_7/utils.py +++ /dev/null @@ -1,124 +0,0 @@ -import hashlib -import secrets -import bcrypt -import sys -import re - -from . import security - - -class TemplateException(Exception): - pass - - -def throw_error(message): - # When throwing a known error, hide the traceback - # This is because the error is also shown in the UI - # and having a traceback makes it hard for user to read - sys.tracebacklimit = 0 - raise TemplateException(message) - - -def secure_string(length): - return secrets.token_urlsafe(length) - - -def basic_auth_header(username, password): - return f"Basic {security.basic_auth(username, password)}" - - -def bcrypt_hash(password): - return bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - - -def match_regex(value, regex): - if not re.match(regex, value): - return False - return True - - -def must_match_regex(value, regex): - if not match_regex(value, regex): - throw_error(f"Expected [{value}] to match [{regex}]") - return value - - -def merge_dicts(*dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - -# Basic validation for a path (Expand later) -def valid_path(path=""): - if not path.startswith("/"): - throw_error(f"Expected path [{path}] to start with /") - - # There is no reason to allow / as a path, either on host or in a container - if path == "/": - throw_error(f"Expected path [{path}] to not be /") - - return path - - -def camel_case(string): - return string.title() - - -def is_boolean(string): - return string.lower() in ["true", "false"] - - -def is_number(string): - try: - float(string) - return True - except ValueError: - return False - - -def get_image(images={}, name=""): - if not images: - throw_error("Expected [images] to be set") - if name not in images: - throw_error(f"Expected [images.{name}] to be set") - if not images[name].get("repository") or not images[name].get("tag"): - throw_error(f"Expected [images.{name}.repository] and [images.{name}.tag] to be set") - - return f"{images[name]['repository']}:{images[name]['tag']}" - - -def hash_data(data=""): - if not data: - throw_error("Expected [data] to be set") - return hashlib.sha256(data.encode("utf-8")).hexdigest() - - -def get_image_with_hashed_data(images={}, name="", data=""): - return f"ix-{get_image(images, name)}-{hash_data(data)}" - - -def copy_dict(dict): - return dict.copy() - - -def escape_dollar(text: str) -> str: - return text.replace("$", "$$") - - -def auto_cast(value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value diff --git a/ix-dev/community/whoogle/templates/library/base_v1_1_7/__init__.py b/ix-dev/community/whoogle/templates/library/base_v2_0_30/__init__.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v1_1_7/__init__.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_30/__init__.py diff --git a/library/2.0.30/configs.py b/ix-dev/community/whoogle/templates/library/base_v2_0_30/configs.py similarity index 100% rename from library/2.0.30/configs.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_30/configs.py diff --git a/library/2.0.30/container.py b/ix-dev/community/whoogle/templates/library/base_v2_0_30/container.py similarity index 100% rename from library/2.0.30/container.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_30/container.py diff --git a/library/2.0.30/depends.py b/ix-dev/community/whoogle/templates/library/base_v2_0_30/depends.py similarity index 100% rename from library/2.0.30/depends.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_30/depends.py diff --git a/library/2.0.30/deploy.py b/ix-dev/community/whoogle/templates/library/base_v2_0_30/deploy.py similarity index 100% rename from library/2.0.30/deploy.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_30/deploy.py diff --git a/library/2.0.30/deps.py b/ix-dev/community/whoogle/templates/library/base_v2_0_30/deps.py similarity index 100% rename from library/2.0.30/deps.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_30/deps.py diff --git a/library/2.0.30/device.py b/ix-dev/community/whoogle/templates/library/base_v2_0_30/device.py similarity index 100% rename from library/2.0.30/device.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_30/device.py diff --git a/library/2.0.30/devices.py b/ix-dev/community/whoogle/templates/library/base_v2_0_30/devices.py similarity index 100% rename from library/2.0.30/devices.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_30/devices.py diff --git a/library/2.0.30/dns.py b/ix-dev/community/whoogle/templates/library/base_v2_0_30/dns.py similarity index 100% rename from library/2.0.30/dns.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_30/dns.py diff --git a/library/2.0.30/environment.py b/ix-dev/community/whoogle/templates/library/base_v2_0_30/environment.py similarity index 100% rename from library/2.0.30/environment.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_30/environment.py diff --git a/library/2.0.30/error.py b/ix-dev/community/whoogle/templates/library/base_v2_0_30/error.py similarity index 100% rename from library/2.0.30/error.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_30/error.py diff --git a/library/2.0.30/formatter.py b/ix-dev/community/whoogle/templates/library/base_v2_0_30/formatter.py similarity index 100% rename from library/2.0.30/formatter.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_30/formatter.py diff --git a/library/2.0.30/functions.py b/ix-dev/community/whoogle/templates/library/base_v2_0_30/functions.py similarity index 94% rename from library/2.0.30/functions.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_30/functions.py index b0c834ab4b..47a2c1233a 100644 --- a/library/2.0.30/functions.py +++ b/ix-dev/community/whoogle/templates/library/base_v2_0_30/functions.py @@ -98,6 +98,11 @@ def _or_default(self, value, default): return default return value + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + def _get_host_path(self, storage): source_type = storage.get("type", "") if not source_type: @@ -139,4 +144,5 @@ def func_map(self): "disallow_chars": self._disallow_chars, "get_host_path": self._get_host_path, "or_default": self._or_default, + "temp_config": self._temp_config, } diff --git a/library/2.0.30/healthcheck.py b/ix-dev/community/whoogle/templates/library/base_v2_0_30/healthcheck.py similarity index 100% rename from library/2.0.30/healthcheck.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_30/healthcheck.py diff --git a/library/2.0.30/labels.py b/ix-dev/community/whoogle/templates/library/base_v2_0_30/labels.py similarity index 100% rename from library/2.0.30/labels.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_30/labels.py diff --git a/library/2.0.30/notes.py b/ix-dev/community/whoogle/templates/library/base_v2_0_30/notes.py similarity index 100% rename from library/2.0.30/notes.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_30/notes.py diff --git a/library/2.0.30/portal.py b/ix-dev/community/whoogle/templates/library/base_v2_0_30/portal.py similarity index 100% rename from library/2.0.30/portal.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_30/portal.py diff --git a/library/2.0.30/portals.py b/ix-dev/community/whoogle/templates/library/base_v2_0_30/portals.py similarity index 100% rename from library/2.0.30/portals.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_30/portals.py diff --git a/library/2.0.30/ports.py b/ix-dev/community/whoogle/templates/library/base_v2_0_30/ports.py similarity index 100% rename from library/2.0.30/ports.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_30/ports.py diff --git a/library/2.0.30/render.py b/ix-dev/community/whoogle/templates/library/base_v2_0_30/render.py similarity index 100% rename from library/2.0.30/render.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_30/render.py diff --git a/library/2.0.30/resources.py b/ix-dev/community/whoogle/templates/library/base_v2_0_30/resources.py similarity index 100% rename from library/2.0.30/resources.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_30/resources.py diff --git a/library/2.0.30/restart.py b/ix-dev/community/whoogle/templates/library/base_v2_0_30/restart.py similarity index 100% rename from library/2.0.30/restart.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_30/restart.py diff --git a/library/2.0.30/storage.py b/ix-dev/community/whoogle/templates/library/base_v2_0_30/storage.py similarity index 100% rename from library/2.0.30/storage.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_30/storage.py diff --git a/library/2.0.30/sysctls.py b/ix-dev/community/whoogle/templates/library/base_v2_0_30/sysctls.py similarity index 100% rename from library/2.0.30/sysctls.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_30/sysctls.py diff --git a/library/2.0.30/__init__.py b/ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/__init__.py similarity index 100% rename from library/2.0.30/__init__.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/__init__.py diff --git a/library/2.0.30/tests/test_build_image.py b/ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_build_image.py similarity index 100% rename from library/2.0.30/tests/test_build_image.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_build_image.py diff --git a/library/2.0.30/tests/test_configs.py b/ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_configs.py similarity index 100% rename from library/2.0.30/tests/test_configs.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_configs.py diff --git a/library/2.0.30/tests/test_container.py b/ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_container.py similarity index 100% rename from library/2.0.30/tests/test_container.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_container.py diff --git a/library/2.0.30/tests/test_depends.py b/ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_depends.py similarity index 100% rename from library/2.0.30/tests/test_depends.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_depends.py diff --git a/library/2.0.30/tests/test_deps.py b/ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_deps.py similarity index 100% rename from library/2.0.30/tests/test_deps.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_deps.py diff --git a/library/2.0.30/tests/test_device.py b/ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_device.py similarity index 100% rename from library/2.0.30/tests/test_device.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_device.py diff --git a/library/2.0.30/tests/test_dns.py b/ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_dns.py similarity index 100% rename from library/2.0.30/tests/test_dns.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_dns.py diff --git a/library/2.0.30/tests/test_environment.py b/ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_environment.py similarity index 100% rename from library/2.0.30/tests/test_environment.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_environment.py diff --git a/library/2.0.30/tests/test_formatter.py b/ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_formatter.py similarity index 100% rename from library/2.0.30/tests/test_formatter.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_formatter.py diff --git a/library/2.0.30/tests/test_functions.py b/ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_functions.py similarity index 93% rename from library/2.0.30/tests/test_functions.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_functions.py index 13d5e49522..c7e250fcf0 100644 --- a/library/2.0.30/tests/test_functions.py +++ b/ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_functions.py @@ -66,6 +66,12 @@ def test_funcs(mock_values): {"func": "or_default", "values": [1, None], "expected": 1}, {"func": "or_default", "values": [False, 1], "expected": 1}, {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expected_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"name": "test"}}, + }, ] for test in tests: diff --git a/library/2.0.30/tests/test_healthcheck.py b/ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_healthcheck.py similarity index 100% rename from library/2.0.30/tests/test_healthcheck.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_healthcheck.py diff --git a/library/2.0.30/tests/test_labels.py b/ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_labels.py similarity index 100% rename from library/2.0.30/tests/test_labels.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_labels.py diff --git a/library/2.0.30/tests/test_notes.py b/ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_notes.py similarity index 100% rename from library/2.0.30/tests/test_notes.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_notes.py diff --git a/library/2.0.30/tests/test_portal.py b/ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_portal.py similarity index 100% rename from library/2.0.30/tests/test_portal.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_portal.py diff --git a/library/2.0.30/tests/test_ports.py b/ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_ports.py similarity index 100% rename from library/2.0.30/tests/test_ports.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_ports.py diff --git a/library/2.0.30/tests/test_render.py b/ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_render.py similarity index 100% rename from library/2.0.30/tests/test_render.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_render.py diff --git a/library/2.0.30/tests/test_resources.py b/ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_resources.py similarity index 100% rename from library/2.0.30/tests/test_resources.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_resources.py diff --git a/library/2.0.30/tests/test_restart.py b/ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_restart.py similarity index 100% rename from library/2.0.30/tests/test_restart.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_restart.py diff --git a/library/2.0.30/tests/test_sysctls.py b/ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_sysctls.py similarity index 100% rename from library/2.0.30/tests/test_sysctls.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_sysctls.py diff --git a/library/2.0.30/tests/test_volumes.py b/ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_volumes.py similarity index 100% rename from library/2.0.30/tests/test_volumes.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_volumes.py diff --git a/library/2.0.30/validations.py b/ix-dev/community/whoogle/templates/library/base_v2_0_30/validations.py similarity index 100% rename from library/2.0.30/validations.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_30/validations.py diff --git a/library/2.0.30/volume_mount.py b/ix-dev/community/whoogle/templates/library/base_v2_0_30/volume_mount.py similarity index 100% rename from library/2.0.30/volume_mount.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_30/volume_mount.py diff --git a/library/2.0.30/volume_mount_types.py b/ix-dev/community/whoogle/templates/library/base_v2_0_30/volume_mount_types.py similarity index 100% rename from library/2.0.30/volume_mount_types.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_30/volume_mount_types.py diff --git a/library/2.0.30/volume_sources.py b/ix-dev/community/whoogle/templates/library/base_v2_0_30/volume_sources.py similarity index 100% rename from library/2.0.30/volume_sources.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_30/volume_sources.py diff --git a/library/2.0.30/volume_types.py b/ix-dev/community/whoogle/templates/library/base_v2_0_30/volume_types.py similarity index 100% rename from library/2.0.30/volume_types.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_30/volume_types.py diff --git a/library/2.0.30/volumes.py b/ix-dev/community/whoogle/templates/library/base_v2_0_30/volumes.py similarity index 100% rename from library/2.0.30/volumes.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_30/volumes.py diff --git a/library/2.0.30/tests/__init__.py b/library/2.0.31/__init__.py similarity index 100% rename from library/2.0.30/tests/__init__.py rename to library/2.0.31/__init__.py diff --git a/library/2.0.31/configs.py b/library/2.0.31/configs.py new file mode 100644 index 0000000000..b76f4b169c --- /dev/null +++ b/library/2.0.31/configs.py @@ -0,0 +1,86 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class Configs: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._configs: dict[str, dict] = {} + + def add(self, name: str, data: str): + if not isinstance(data, str): + raise RenderError(f"Expected [data] to be a string, got [{type(data)}]") + + if name not in self._configs: + self._configs[name] = {"name": name, "data": data} + return + + if data == self._configs[name]["data"]: + return + + raise RenderError(f"Config [{name}] already added with different data") + + def has_configs(self): + return bool(self._configs) + + def render(self): + return { + c["name"]: {"content": escape_dollar(c["data"])} + for c in sorted(self._configs.values(), key=lambda c: c["name"]) + } + + +class ContainerConfigs: + def __init__(self, render_instance: "Render", configs: Configs): + self._render_instance = render_instance + self.top_level_configs: Configs = configs + self.container_configs: set[ContainerConfig] = set() + + def add(self, name: str, data: str, target: str, mode: str = ""): + self.top_level_configs.add(name, data) + + if target == "": + raise RenderError(f"Expected [target] to be set for config [{name}]") + if mode != "": + mode = valid_octal_mode_or_raise(mode) + + if target in [c.target for c in self.container_configs]: + raise RenderError(f"Target [{target}] already used for another config") + target = valid_fs_path_or_raise(target) + self.container_configs.add(ContainerConfig(self._render_instance, name, target, mode)) + + def has_configs(self): + return bool(self.container_configs) + + def render(self): + return [c.render() for c in sorted(self.container_configs, key=lambda c: c.source)] + + +class ContainerConfig: + def __init__(self, render_instance: "Render", source: str, target: str, mode: str): + self._render_instance = render_instance + self.source = source + self.target = target + self.mode = mode + + def render(self): + result: dict[str, str | int] = { + "source": self.source, + "target": self.target, + } + + if self.mode: + result["mode"] = int(self.mode, 8) + + return result diff --git a/library/2.0.31/container.py b/library/2.0.31/container.py new file mode 100644 index 0000000000..a95e76734c --- /dev/null +++ b/library/2.0.31/container.py @@ -0,0 +1,317 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + "cap_drop": sorted(self._cap_drop), + "healthcheck": self.healthcheck.render(), + } + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/library/2.0.31/depends.py b/library/2.0.31/depends.py new file mode 100644 index 0000000000..4e057cf085 --- /dev/null +++ b/library/2.0.31/depends.py @@ -0,0 +1,34 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_depend_condition_or_raise +except ImportError: + from error import RenderError + from validations import valid_depend_condition_or_raise + + +class Depends: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._dependencies: dict[str, str] = {} + + def add_dependency(self, name: str, condition: str): + condition = valid_depend_condition_or_raise(condition) + if name in self._dependencies.keys(): + raise RenderError(f"Dependency [{name}] already added") + if name not in self._render_instance.container_names(): + raise RenderError( + f"Dependency [{name}] not found in defined containers. " + f"Available containers: [{', '.join(self._render_instance.container_names())}]" + ) + self._dependencies[name] = condition + + def has_dependencies(self): + return len(self._dependencies) > 0 + + def render(self): + return {d: {"condition": c} for d, c in self._dependencies.items()} diff --git a/library/2.0.31/deploy.py b/library/2.0.31/deploy.py new file mode 100644 index 0000000000..894dbc643b --- /dev/null +++ b/library/2.0.31/deploy.py @@ -0,0 +1,24 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .resources import Resources +except ImportError: + from resources import Resources + + +class Deploy: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self.resources: Resources = Resources(self._render_instance) + + def has_deploy(self): + return self.resources.has_resources() + + def render(self): + if self.resources.has_resources(): + return {"resources": self.resources.render()} + + return {} diff --git a/library/2.0.31/deps.py b/library/2.0.31/deps.py new file mode 100644 index 0000000000..b3607fa6ab --- /dev/null +++ b/library/2.0.31/deps.py @@ -0,0 +1,454 @@ +import os +import json +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import ( + valid_port_or_raise, + valid_fs_path_or_raise, + valid_octal_mode_or_raise, + valid_redis_password_or_raise, + ) +except ImportError: + from error import RenderError + from validations import ( + valid_port_or_raise, + valid_fs_path_or_raise, + valid_octal_mode_or_raise, + valid_redis_password_or_raise, + ) + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = os.path.join("/mnt/permission", identifier) + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + + c.add_storage("/var/lib/postgresql/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("POSTGRES_USER", config["user"]) + c.environment.add_env("POSTGRES_PASSWORD", config["password"]) + c.environment.add_env("POSTGRES_DB", config["database"]) + c.environment.add_env("POSTGRES_PORT", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def _get_port(self): + return self._config.get("port") or 5432 + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/library/2.0.31/device.py b/library/2.0.31/device.py new file mode 100644 index 0000000000..bfe97097cb --- /dev/null +++ b/library/2.0.31/device.py @@ -0,0 +1,31 @@ +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise, allowed_device_or_raise, valid_cgroup_perm_or_raise +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise, allowed_device_or_raise, valid_cgroup_perm_or_raise + + +class Device: + def __init__(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + hd = valid_fs_path_or_raise(host_device.rstrip("/")) + cd = valid_fs_path_or_raise(container_device.rstrip("/")) + if not hd or not cd: + raise RenderError( + "Expected [host_device] and [container_device] to be set. " + f"Got host_device [{host_device}] and container_device [{container_device}]" + ) + + cgroup_perm = valid_cgroup_perm_or_raise(cgroup_perm) + if not allow_disallowed: + hd = allowed_device_or_raise(hd) + + self.cgroup_perm: str = cgroup_perm + self.host_device: str = hd + self.container_device: str = cd + + def render(self): + result = f"{self.host_device}:{self.container_device}" + if self.cgroup_perm: + result += f":{self.cgroup_perm}" + return result diff --git a/library/2.0.31/devices.py b/library/2.0.31/devices.py new file mode 100644 index 0000000000..ae22c79d2e --- /dev/null +++ b/library/2.0.31/devices.py @@ -0,0 +1,66 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/library/2.0.31/dns.py b/library/2.0.31/dns.py new file mode 100644 index 0000000000..d3ae7b19fa --- /dev/null +++ b/library/2.0.31/dns.py @@ -0,0 +1,79 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import allowed_dns_opt_or_raise +except ImportError: + from error import RenderError + from validations import allowed_dns_opt_or_raise + + +class Dns: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._dns_options: set[str] = set() + self._dns_searches: set[str] = set() + self._dns_nameservers: set[str] = set() + + self._auto_add_dns_opts_from_values() + self._auto_add_dns_searches_from_values() + self._auto_add_dns_nameservers_from_values() + + def _get_dns_opt_keys(self): + return [self._get_key_from_opt(opt) for opt in self._dns_options] + + def _get_key_from_opt(self, opt): + return opt.split(":")[0] + + def _auto_add_dns_opts_from_values(self): + values = self._render_instance.values + for dns_opt in values.get("network", {}).get("dns_opts", []): + self.add_dns_opt(dns_opt) + + def _auto_add_dns_searches_from_values(self): + values = self._render_instance.values + for dns_search in values.get("network", {}).get("dns_searches", []): + self.add_dns_search(dns_search) + + def _auto_add_dns_nameservers_from_values(self): + values = self._render_instance.values + for dns_nameserver in values.get("network", {}).get("dns_nameservers", []): + self.add_dns_nameserver(dns_nameserver) + + def add_dns_search(self, dns_search): + if dns_search in self._dns_searches: + raise RenderError(f"DNS Search [{dns_search}] already added") + self._dns_searches.add(dns_search) + + def add_dns_nameserver(self, dns_nameserver): + if dns_nameserver in self._dns_nameservers: + raise RenderError(f"DNS Nameserver [{dns_nameserver}] already added") + self._dns_nameservers.add(dns_nameserver) + + def add_dns_opt(self, dns_opt): + # eg attempts:3 + key = allowed_dns_opt_or_raise(self._get_key_from_opt(dns_opt)) + if key in self._get_dns_opt_keys(): + raise RenderError(f"DNS Option [{key}] already added") + self._dns_options.add(dns_opt) + + def has_dns_opts(self): + return len(self._dns_options) > 0 + + def has_dns_searches(self): + return len(self._dns_searches) > 0 + + def has_dns_nameservers(self): + return len(self._dns_nameservers) > 0 + + def render_dns_searches(self): + return sorted(self._dns_searches) + + def render_dns_opts(self): + return sorted(self._dns_options) + + def render_dns_nameservers(self): + return sorted(self._dns_nameservers) diff --git a/library/2.0.31/environment.py b/library/2.0.31/environment.py new file mode 100644 index 0000000000..850a3afd8e --- /dev/null +++ b/library/2.0.31/environment.py @@ -0,0 +1,109 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/library/2.0.31/error.py b/library/2.0.31/error.py new file mode 100644 index 0000000000..aef48d3b02 --- /dev/null +++ b/library/2.0.31/error.py @@ -0,0 +1,4 @@ +class RenderError(Exception): + """Base class for exceptions in this module.""" + + pass diff --git a/library/2.0.31/formatter.py b/library/2.0.31/formatter.py new file mode 100644 index 0000000000..24e882f47a --- /dev/null +++ b/library/2.0.31/formatter.py @@ -0,0 +1,26 @@ +import json +import hashlib + + +def escape_dollar(text: str) -> str: + return text.replace("$", "$$") + + +def get_hashed_name_for_volume(prefix: str, config: dict): + config_hash = hashlib.sha256(json.dumps(config).encode("utf-8")).hexdigest() + return f"{prefix}_{config_hash}" + + +def get_hash_with_prefix(prefix: str, data: str): + return f"{prefix}_{hashlib.sha256(data.encode('utf-8')).hexdigest()}" + + +def merge_dicts_no_overwrite(dict1, dict2): + overlapping_keys = dict1.keys() & dict2.keys() + if overlapping_keys: + raise ValueError(f"Merging of dicts failed. Overlapping keys: {overlapping_keys}") + return {**dict1, **dict2} + + +def get_image_with_hashed_data(image: str, data: str): + return get_hash_with_prefix(f"ix-{image}", data) diff --git a/library/2.0.31/functions.py b/library/2.0.31/functions.py new file mode 100644 index 0000000000..47a2c1233a --- /dev/null +++ b/library/2.0.31/functions.py @@ -0,0 +1,148 @@ +import re +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return dict.copy() + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/library/2.0.31/healthcheck.py b/library/2.0.31/healthcheck.py new file mode 100644 index 0000000000..36ae5d90aa --- /dev/null +++ b/library/2.0.31/healthcheck.py @@ -0,0 +1,193 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def render(self): + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/library/2.0.31/labels.py b/library/2.0.31/labels.py new file mode 100644 index 0000000000..f1e667ba00 --- /dev/null +++ b/library/2.0.31/labels.py @@ -0,0 +1,37 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar +except ImportError: + from error import RenderError + from formatter import escape_dollar + + +class Labels: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._labels: dict[str, str] = {} + + def add_label(self, key: str, value: str): + if not key: + raise RenderError("Labels must have a key") + + if key.startswith("com.docker.compose"): + raise RenderError(f"Label [{key}] cannot start with [com.docker.compose] as it is reserved") + + if key in self._labels.keys(): + raise RenderError(f"Label [{key}] already added") + + self._labels[key] = escape_dollar(str(value)) + + def has_labels(self) -> bool: + return bool(self._labels) + + def render(self) -> dict[str, str]: + if not self.has_labels(): + return {} + return {label: value for label, value in sorted(self._labels.items())} diff --git a/library/2.0.31/notes.py b/library/2.0.31/notes.py new file mode 100644 index 0000000000..4adc50c3d8 --- /dev/null +++ b/library/2.0.31/notes.py @@ -0,0 +1,70 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + + +class Notes: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._app_name: str = "" + self._warnings: list[str] = [] + self._deprecations: list[str] = [] + self._header: str = "" + self._body: str = "" + self._footer: str = "" + + self._auto_set_app_name() + self._auto_set_header() + self._auto_set_footer() + + def _auto_set_app_name(self): + app_name = self._render_instance.values.get("ix_context", {}).get("app_metadata", {}).get("name", "") + self._app_name = app_name or "" + + def _auto_set_header(self): + head = "# Welcome to TrueNAS SCALE\n\n" + head += f"Thank you for installing {self._app_name}!\n\n" + self._header = head + + def _auto_set_footer(self): + footer = "## Documentation\n\n" + footer += f"Documentation for {self._app_name} can be found at https://www.truenas.com/docs.\n\n" + footer += "## Bug reports\n\n" + footer += "If you find a bug in this app, please file an issue at\n" + footer += "https://ixsystems.atlassian.net or https://github.com/truenas/apps\n\n" + footer += "## Feature requests or improvements\n\n" + footer += "If you find a feature request for this app, please file an issue at\n" + footer += "https://ixsystems.atlassian.net or https://github.com/truenas/apps\n" + self._footer = footer + + def add_warning(self, warning: str): + self._warnings.append(warning) + + def add_deprecation(self, deprecation: str): + self._deprecations.append(deprecation) + + def set_body(self, body: str): + self._body = body + + def render(self): + result = self._header + + if self._warnings: + result += "## Warnings\n\n" + for warning in self._warnings: + result += f"- {warning}\n" + result += "\n" + + if self._deprecations: + result += "## Deprecations\n\n" + for deprecation in self._deprecations: + result += f"- {deprecation}\n" + result += "\n" + + if self._body: + result += self._body.strip() + "\n\n" + + result += self._footer + + return result diff --git a/library/2.0.31/portal.py b/library/2.0.31/portal.py new file mode 100644 index 0000000000..cf47163439 --- /dev/null +++ b/library/2.0.31/portal.py @@ -0,0 +1,22 @@ +try: + from .validations import valid_portal_scheme_or_raise, valid_http_path_or_raise, valid_port_or_raise +except ImportError: + from validations import valid_portal_scheme_or_raise, valid_http_path_or_raise, valid_port_or_raise + + +class Portal: + def __init__(self, name: str, config: dict): + self._name = name + self._scheme = valid_portal_scheme_or_raise(config.get("scheme", "http")) + self._host = config.get("host", "0.0.0.0") or "0.0.0.0" + self._port = valid_port_or_raise(config.get("port", 0)) + self._path = valid_http_path_or_raise(config.get("path", "/")) + + def render(self): + return { + "name": self._name, + "scheme": self._scheme, + "host": self._host, + "port": self._port, + "path": self._path, + } diff --git a/library/2.0.31/portals.py b/library/2.0.31/portals.py new file mode 100644 index 0000000000..e106d231e6 --- /dev/null +++ b/library/2.0.31/portals.py @@ -0,0 +1,28 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .portal import Portal +except ImportError: + from error import RenderError + from portal import Portal + + +class Portals: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._portals: set[Portal] = set() + + def add_portal(self, config: dict): + name = config.get("name", "Web UI") + + if name in [p._name for p in self._portals]: + raise RenderError(f"Portal [{name}] already added") + + self._portals.add(Portal(name, config)) + + def render(self): + return [p.render() for _, p in sorted([(p._name, p) for p in self._portals])] diff --git a/library/2.0.31/ports.py b/library/2.0.31/ports.py new file mode 100644 index 0000000000..f11e1481b4 --- /dev/null +++ b/library/2.0.31/ports.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import ( + valid_port_or_raise, + valid_port_protocol_or_raise, + valid_port_mode_or_raise, + valid_ip_or_raise, + ) +except ImportError: + from error import RenderError + from validations import ( + valid_port_or_raise, + valid_port_protocol_or_raise, + valid_port_mode_or_raise, + valid_ip_or_raise, + ) + + +class Ports: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._ports: dict[str, dict] = {} + + def add_port(self, host_port: int, container_port: int, config: dict | None = None): + config = config or {} + host_port = valid_port_or_raise(host_port) + container_port = valid_port_or_raise(container_port) + proto = valid_port_protocol_or_raise(config.get("protocol", "tcp")) + mode = valid_port_mode_or_raise(config.get("mode", "ingress")) + host_ip = valid_ip_or_raise(config.get("host_ip", "0.0.0.0")) + + key = f"{host_port}_{host_ip}_{proto}" + if key in self._ports.keys(): + raise RenderError(f"Port [{host_port}/{proto}] already added for [{host_ip}]") + + if host_ip != "0.0.0.0": + # If the port we are adding is not going to use 0.0.0.0 + # Make sure that we don't have already added that port/proto to 0.0.0.0 + search_key = f"{host_port}_0.0.0.0_{proto}" + if search_key in self._ports.keys(): + raise RenderError(f"Cannot bind port [{host_port}/{proto}] to [{host_ip}], already bound to [0.0.0.0]") + elif host_ip == "0.0.0.0": + # If the port we are adding is going to use 0.0.0.0 + # Make sure that we don't have already added that port/proto to a specific ip + for p in self._ports.values(): + if p["published"] == host_port and p["protocol"] == proto: + raise RenderError( + f"Cannot bind port [{host_port}/{proto}] to [{host_ip}], already bound to [{p['host_ip']}]" + ) + + self._ports[key] = { + "published": host_port, + "target": container_port, + "protocol": proto, + "mode": mode, + "host_ip": host_ip, + } + + def has_ports(self): + return len(self._ports) > 0 + + def render(self): + return [config for _, config in sorted(self._ports.items())] diff --git a/library/2.0.31/render.py b/library/2.0.31/render.py new file mode 100644 index 0000000000..9d8fcc28d5 --- /dev/null +++ b/library/2.0.31/render.py @@ -0,0 +1,89 @@ +import copy + +try: + from .configs import Configs + from .container import Container + from .deps import Deps + from .error import RenderError + from .functions import Functions + from .notes import Notes + from .portals import Portals + from .volumes import Volumes +except ImportError: + from configs import Configs + from container import Container + from deps import Deps + from error import RenderError + from functions import Functions + from notes import Notes + from portals import Portals + from volumes import Volumes + + +class Render(object): + def __init__(self, values): + self._containers: dict[str, Container] = {} + self.values = values + self._add_images_internal_use() + # Make a copy after we inject the images + self._original_values: dict = copy.deepcopy(self.values) + + self.deps: Deps = Deps(self) + + self.configs = Configs(render_instance=self) + self.funcs = Functions(render_instance=self).func_map() + self.portals: Portals = Portals(render_instance=self) + self.notes: Notes = Notes(render_instance=self) + self.volumes = Volumes(render_instance=self) + + def _add_images_internal_use(self): + if not self.values.get("images"): + self.values["images"] = {} + + if "python_permissions_image" not in self.values["images"]: + self.values["images"]["python_permissions_image"] = {"repository": "python", "tag": "3.13.0-slim-bookworm"} + + def container_names(self): + return list(self._containers.keys()) + + def add_container(self, name: str, image: str): + name = name.strip() + if not name: + raise RenderError("Container name cannot be empty") + container = Container(self, name, image) + if name in self._containers: + raise RenderError(f"Container {name} already exists.") + self._containers[name] = container + return container + + def render(self): + if self.values != self._original_values: + raise RenderError("Values have been modified since the renderer was created.") + + if not self._containers: + raise RenderError("No containers added.") + + result: dict = { + "x-notes": self.notes.render(), + "x-portals": self.portals.render(), + "services": {c._name: c.render() for c in self._containers.values()}, + } + + # Make sure that after services are rendered + # there are no labels that target a non-existent container + # This is to prevent typos + for label in self.values.get("labels", []): + for c in label.get("containers", []): + if c not in self.container_names(): + raise RenderError(f"Label [{label['key']}] references container [{c}] which does not exist") + + if self.volumes.has_volumes(): + result["volumes"] = self.volumes.render() + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + # if self.networks: + # result["networks"] = {...} + + return result diff --git a/library/2.0.31/resources.py b/library/2.0.31/resources.py new file mode 100644 index 0000000000..733f43bb6f --- /dev/null +++ b/library/2.0.31/resources.py @@ -0,0 +1,115 @@ +import re +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +DEFAULT_CPUS = 2.0 +DEFAULT_MEMORY = 4096 + + +class Resources: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._limits: dict = {} + self._reservations: dict = {} + self._nvidia_ids: set[str] = set() + self._auto_add_cpu_from_values() + self._auto_add_memory_from_values() + self._auto_add_gpus_from_values() + + def _set_cpu(self, cpus: Any): + c = str(cpus) + if not re.match(r"^[1-9][0-9]*(\.[0-9]+)?$", c): + raise RenderError(f"Expected cpus to be a number or a float (minimum 1.0), got [{cpus}]") + self._limits.update({"cpus": c}) + + def _set_memory(self, memory: Any): + m = str(memory) + if not re.match(r"^[1-9][0-9]*$", m): + raise RenderError(f"Expected memory to be a number, got [{memory}]") + self._limits.update({"memory": f"{m}M"}) + + def _auto_add_cpu_from_values(self): + resources = self._render_instance.values.get("resources", {}) + self._set_cpu(resources.get("limits", {}).get("cpus", DEFAULT_CPUS)) + + def _auto_add_memory_from_values(self): + resources = self._render_instance.values.get("resources", {}) + self._set_memory(resources.get("limits", {}).get("memory", DEFAULT_MEMORY)) + + def _auto_add_gpus_from_values(self): + resources = self._render_instance.values.get("resources", {}) + gpus = resources.get("gpus", {}).get("nvidia_gpu_selection", {}) + if not gpus: + return + + for pci, gpu in gpus.items(): + if gpu.get("use_gpu", False): + if not gpu.get("uuid"): + raise RenderError(f"Expected [uuid] to be set for GPU in slot [{pci}] in [nvidia_gpu_selection]") + self._nvidia_ids.add(gpu["uuid"]) + + if self._nvidia_ids: + if not self._reservations: + self._reservations["devices"] = [] + self._reservations["devices"].append( + { + "capabilities": ["gpu"], + "driver": "nvidia", + "device_ids": sorted(self._nvidia_ids), + } + ) + + # This is only used on ix-app that we allow + # disabling cpus and memory. GPUs are only added + # if the user has requested them. + def remove_cpus_and_memory(self): + self._limits.pop("cpus", None) + self._limits.pop("memory", None) + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._reservations.pop("devices", None) + + def set_profile(self, profile: str): + cpu, memory = profile_mapping(profile) + self._set_cpu(cpu) + self._set_memory(memory) + + def has_resources(self): + return len(self._limits) > 0 or len(self._reservations) > 0 + + def has_gpus(self): + gpu_devices = [d for d in self._reservations.get("devices", []) if "gpu" in d["capabilities"]] + return len(gpu_devices) > 0 + + def render(self): + result = {} + if self._limits: + result["limits"] = self._limits + if self._reservations: + result["reservations"] = self._reservations + + return result + + +def profile_mapping(profile: str): + profiles = { + "low": (1, 512), + "medium": (2, 1024), + } + + if profile not in profiles: + raise RenderError( + f"Resource profile [{profile}] is not valid. Valid options are: [{', '.join(profiles.keys())}]" + ) + + return profiles[profile] diff --git a/library/2.0.31/restart.py b/library/2.0.31/restart.py new file mode 100644 index 0000000000..2f6281af48 --- /dev/null +++ b/library/2.0.31/restart.py @@ -0,0 +1,25 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .validations import valid_restart_policy_or_raise +except ImportError: + from validations import valid_restart_policy_or_raise + + +class RestartPolicy: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._policy: str = "unless-stopped" + self._maximum_retry_count: int = 0 + + def set_policy(self, policy: str, maximum_retry_count: int = 0): + self._policy = valid_restart_policy_or_raise(policy, maximum_retry_count) + self._maximum_retry_count = maximum_retry_count + + def render(self): + if self._policy == "on-failure" and self._maximum_retry_count > 0: + return f"{self._policy}:{self._maximum_retry_count}" + return self._policy diff --git a/library/2.0.31/storage.py b/library/2.0.31/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/library/2.0.31/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/library/2.0.31/sysctls.py b/library/2.0.31/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/library/2.0.31/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/library/2.0.31/tests/__init__.py b/library/2.0.31/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/library/2.0.31/tests/test_build_image.py b/library/2.0.31/tests/test_build_image.py new file mode 100644 index 0000000000..f30c1210ed --- /dev/null +++ b/library/2.0.31/tests/test_build_image.py @@ -0,0 +1,49 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_build_image_with_from(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.build_image(["FROM test_image"]) + + +def test_build_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.build_image( + [ + "RUN echo hello", + None, + "", + "RUN echo world", + ] + ) + output = render.render() + assert ( + output["services"]["test_container"]["image"] + == "ix-nginx:latest_4a127145ea4c25511707e57005dd0ed457fe2f4932082c8f9faa339a450b6a99" + ) + assert output["services"]["test_container"]["build"] == { + "tags": ["ix-nginx:latest_4a127145ea4c25511707e57005dd0ed457fe2f4932082c8f9faa339a450b6a99"], + "dockerfile_inline": """FROM nginx:latest +RUN echo hello +RUN echo world +""", + } diff --git a/library/2.0.31/tests/test_configs.py b/library/2.0.31/tests/test_configs.py new file mode 100644 index 0000000000..9049e473ea --- /dev/null +++ b/library/2.0.31/tests/test_configs.py @@ -0,0 +1,63 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_duplicate_config_with_different_data(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.configs.add("test_config", "test_data", "/some/path") + with pytest.raises(Exception): + c1.configs.add("test_config", "test_data2", "/some/path") + + +def test_add_config_with_empty_target(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.configs.add("test_config", "test_data", "") + + +def test_add_duplicate_target(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.configs.add("test_config", "test_data", "/some/path") + with pytest.raises(Exception): + c1.configs.add("test_config2", "test_data2", "/some/path") + + +def test_add_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.configs.add("test_config", "$test_data", "/some/path") + output = render.render() + assert output["configs"]["test_config"]["content"] == "$$test_data" + assert output["services"]["test_container"]["configs"] == [{"source": "test_config", "target": "/some/path"}] + + +def test_add_config_with_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.configs.add("test_config", "test_data", "/some/path", "0777") + output = render.render() + assert output["configs"]["test_config"]["content"] == "test_data" + assert output["services"]["test_container"]["configs"] == [ + {"source": "test_config", "target": "/some/path", "mode": 511} + ] diff --git a/library/2.0.31/tests/test_container.py b/library/2.0.31/tests/test_container.py new file mode 100644 index 0000000000..61a22a5df2 --- /dev/null +++ b/library/2.0.31/tests/test_container.py @@ -0,0 +1,324 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/library/2.0.31/tests/test_depends.py b/library/2.0.31/tests/test_depends.py new file mode 100644 index 0000000000..a1d8373927 --- /dev/null +++ b/library/2.0.31/tests/test_depends.py @@ -0,0 +1,54 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_dependency(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c2 = render.add_container("test_container2", "test_image") + c1.healthcheck.disable() + c2.healthcheck.disable() + c1.depends.add_dependency("test_container2", "service_started") + output = render.render() + assert output["services"]["test_container"]["depends_on"]["test_container2"] == {"condition": "service_started"} + + +def test_add_dependency_invalid_condition(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + render.add_container("test_container2", "test_image") + with pytest.raises(Exception): + c1.depends.add_dependency("test_container2", "invalid_condition") + + +def test_add_dependency_missing_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.depends.add_dependency("test_container2", "service_started") + + +def test_add_dependency_duplicate(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + render.add_container("test_container2", "test_image") + c1.depends.add_dependency("test_container2", "service_started") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.depends.add_dependency("test_container2", "service_started") diff --git a/library/2.0.31/tests/test_deps.py b/library/2.0.31/tests/test_deps.py new file mode 100644 index 0000000000..f9562ba4f2 --- /dev/null +++ b/library/2.0.31/tests/test_deps.py @@ -0,0 +1,380 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:latest" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] diff --git a/library/2.0.31/tests/test_device.py b/library/2.0.31/tests/test_device.py new file mode 100644 index 0000000000..7455c829f6 --- /dev/null +++ b/library/2.0.31/tests/test_device.py @@ -0,0 +1,121 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/library/2.0.31/tests/test_dns.py b/library/2.0.31/tests/test_dns.py new file mode 100644 index 0000000000..fe6b21e34f --- /dev/null +++ b/library/2.0.31/tests/test_dns.py @@ -0,0 +1,64 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_dns_opts(mock_values): + mock_values["network"] = {"dns_opts": ["attempts:3", "opt1", "opt2"]} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["dns_opt"] == ["attempts:3", "opt1", "opt2"] + + +def test_auto_add_dns_searches(mock_values): + mock_values["network"] = {"dns_searches": ["search1", "search2"]} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["dns_search"] == ["search1", "search2"] + + +def test_auto_add_dns_nameservers(mock_values): + mock_values["network"] = {"dns_nameservers": ["nameserver1", "nameserver2"]} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["dns"] == ["nameserver1", "nameserver2"] + + +def test_add_duplicate_dns_nameservers(mock_values): + mock_values["network"] = {"dns_nameservers": ["nameserver1", "nameserver1"]} + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_add_duplicate_dns_searches(mock_values): + mock_values["network"] = {"dns_searches": ["search1", "search1"]} + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_add_duplicate_dns_opts(mock_values): + mock_values["network"] = {"dns_opts": ["attempts:3", "attempts:5"]} + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") diff --git a/library/2.0.31/tests/test_environment.py b/library/2.0.31/tests/test_environment.py new file mode 100644 index 0000000000..209f67551b --- /dev/null +++ b/library/2.0.31/tests/test_environment.py @@ -0,0 +1,184 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/library/2.0.31/tests/test_formatter.py b/library/2.0.31/tests/test_formatter.py new file mode 100644 index 0000000000..843cf65d2e --- /dev/null +++ b/library/2.0.31/tests/test_formatter.py @@ -0,0 +1,13 @@ +from formatter import escape_dollar + + +def test_escape_dollar(): + cases = [ + {"input": "test", "expected": "test"}, + {"input": "$test", "expected": "$$test"}, + {"input": "$$test", "expected": "$$$$test"}, + {"input": "$$$test", "expected": "$$$$$$test"}, + {"input": "$test$", "expected": "$$test$$"}, + ] + for case in cases: + assert escape_dollar(case["input"]) == case["expected"] diff --git a/library/2.0.31/tests/test_functions.py b/library/2.0.31/tests/test_functions.py new file mode 100644 index 0000000000..c7e250fcf0 --- /dev/null +++ b/library/2.0.31/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expected_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/library/2.0.31/tests/test_healthcheck.py b/library/2.0.31/tests/test_healthcheck.py new file mode 100644 index 0000000000..fbd488ece4 --- /dev/null +++ b/library/2.0.31/tests/test_healthcheck.py @@ -0,0 +1,187 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/library/2.0.31/tests/test_labels.py b/library/2.0.31/tests/test_labels.py new file mode 100644 index 0000000000..ffa21eceac --- /dev/null +++ b/library/2.0.31/tests/test_labels.py @@ -0,0 +1,88 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_disallowed_label(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.labels.add_label("com.docker.compose.service", "test_service") + + +def test_add_duplicate_label(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.labels.add_label("my.custom.label", "test_value") + with pytest.raises(Exception): + c1.labels.add_label("my.custom.label", "test_value1") + + +def test_add_label_on_non_existing_container(mock_values): + mock_values["labels"] = [ + { + "key": "my.custom.label1", + "value": "test_value1", + "containers": ["test_container", "test_container2"], + }, + ] + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.render() + + +def test_add_label(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.labels.add_label("my.custom.label1", "test_value1") + c1.labels.add_label("my.custom.label2", "test_value2") + output = render.render() + assert output["services"]["test_container"]["labels"] == { + "my.custom.label1": "test_value1", + "my.custom.label2": "test_value2", + } + + +def test_auto_add_labels(mock_values): + mock_values["labels"] = [ + { + "key": "my.custom.label1", + "value": "test_value1", + "containers": ["test_container", "test_container2"], + }, + { + "key": "my.custom.label2", + "value": "test_value2", + "containers": ["test_container"], + }, + ] + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c2 = render.add_container("test_container2", "test_image") + c1.healthcheck.disable() + c2.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["labels"] == { + "my.custom.label1": "test_value1", + "my.custom.label2": "test_value2", + } + assert output["services"]["test_container2"]["labels"] == { + "my.custom.label1": "test_value1", + } diff --git a/library/2.0.31/tests/test_notes.py b/library/2.0.31/tests/test_notes.py new file mode 100644 index 0000000000..3613445385 --- /dev/null +++ b/library/2.0.31/tests/test_notes.py @@ -0,0 +1,213 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "ix_context": { + "app_metadata": { + "name": "test_app", + } + }, + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_notes(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert ( + output["x-notes"] + == """# Welcome to TrueNAS SCALE + +Thank you for installing test_app! + +## Documentation + +Documentation for test_app can be found at https://www.truenas.com/docs. + +## Bug reports + +If you find a bug in this app, please file an issue at +https://ixsystems.atlassian.net or https://github.com/truenas/apps + +## Feature requests or improvements + +If you find a feature request for this app, please file an issue at +https://ixsystems.atlassian.net or https://github.com/truenas/apps +""" + ) + + +def test_notes_with_warnings(mock_values): + render = Render(mock_values) + render.notes.add_warning("this is not properly configured. fix it now!") + render.notes.add_warning("that is not properly configured. fix it later!") + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert ( + output["x-notes"] + == """# Welcome to TrueNAS SCALE + +Thank you for installing test_app! + +## Warnings + +- this is not properly configured. fix it now! +- that is not properly configured. fix it later! + +## Documentation + +Documentation for test_app can be found at https://www.truenas.com/docs. + +## Bug reports + +If you find a bug in this app, please file an issue at +https://ixsystems.atlassian.net or https://github.com/truenas/apps + +## Feature requests or improvements + +If you find a feature request for this app, please file an issue at +https://ixsystems.atlassian.net or https://github.com/truenas/apps +""" + ) + + +def test_notes_with_deprecations(mock_values): + render = Render(mock_values) + render.notes.add_deprecation("this is will be removed later. fix it now!") + render.notes.add_deprecation("that is will be removed later. fix it later!") + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert ( + output["x-notes"] + == """# Welcome to TrueNAS SCALE + +Thank you for installing test_app! + +## Deprecations + +- this is will be removed later. fix it now! +- that is will be removed later. fix it later! + +## Documentation + +Documentation for test_app can be found at https://www.truenas.com/docs. + +## Bug reports + +If you find a bug in this app, please file an issue at +https://ixsystems.atlassian.net or https://github.com/truenas/apps + +## Feature requests or improvements + +If you find a feature request for this app, please file an issue at +https://ixsystems.atlassian.net or https://github.com/truenas/apps +""" + ) + + +def test_notes_with_body(mock_values): + render = Render(mock_values) + render.notes.set_body( + """## Additional info + +Some info +some other info. +""" + ) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert ( + output["x-notes"] + == """# Welcome to TrueNAS SCALE + +Thank you for installing test_app! + +## Additional info + +Some info +some other info. + +## Documentation + +Documentation for test_app can be found at https://www.truenas.com/docs. + +## Bug reports + +If you find a bug in this app, please file an issue at +https://ixsystems.atlassian.net or https://github.com/truenas/apps + +## Feature requests or improvements + +If you find a feature request for this app, please file an issue at +https://ixsystems.atlassian.net or https://github.com/truenas/apps +""" + ) + + +def test_notes_all(mock_values): + render = Render(mock_values) + render.notes.add_warning("this is not properly configured. fix it now!") + render.notes.add_warning("that is not properly configured. fix it later!") + render.notes.add_deprecation("this is will be removed later. fix it now!") + render.notes.add_deprecation("that is will be removed later. fix it later!") + render.notes.set_body( + """## Additional info + +Some info +some other info. +""" + ) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert ( + output["x-notes"] + == """# Welcome to TrueNAS SCALE + +Thank you for installing test_app! + +## Warnings + +- this is not properly configured. fix it now! +- that is not properly configured. fix it later! + +## Deprecations + +- this is will be removed later. fix it now! +- that is will be removed later. fix it later! + +## Additional info + +Some info +some other info. + +## Documentation + +Documentation for test_app can be found at https://www.truenas.com/docs. + +## Bug reports + +If you find a bug in this app, please file an issue at +https://ixsystems.atlassian.net or https://github.com/truenas/apps + +## Feature requests or improvements + +If you find a feature request for this app, please file an issue at +https://ixsystems.atlassian.net or https://github.com/truenas/apps +""" + ) diff --git a/library/2.0.31/tests/test_portal.py b/library/2.0.31/tests/test_portal.py new file mode 100644 index 0000000000..aebd9425c9 --- /dev/null +++ b/library/2.0.31/tests/test_portal.py @@ -0,0 +1,75 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_no_portals(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["x-portals"] == [] + + +def test_add_portal(mock_values): + render = Render(mock_values) + render.portals.add_portal({"scheme": "http", "path": "/", "port": 8080}) + render.portals.add_portal({"name": "Other Portal", "scheme": "https", "path": "/", "port": 8443}) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["x-portals"] == [ + {"name": "Other Portal", "scheme": "https", "host": "0.0.0.0", "port": 8443, "path": "/"}, + {"name": "Web UI", "scheme": "http", "host": "0.0.0.0", "port": 8080, "path": "/"}, + ] + + +def test_add_duplicate_portal(mock_values): + render = Render(mock_values) + render.portals.add_portal({"scheme": "http", "path": "/", "port": 8080}) + with pytest.raises(Exception): + render.portals.add_portal({"scheme": "http", "path": "/", "port": 8080}) + + +def test_add_duplicate_portal_with_explicit_name(mock_values): + render = Render(mock_values) + render.portals.add_portal({"name": "Some Portal", "scheme": "http", "path": "/", "port": 8080}) + with pytest.raises(Exception): + render.portals.add_portal({"name": "Some Portal", "scheme": "http", "path": "/", "port": 8080}) + + +def test_add_portal_with_invalid_scheme(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.portals.add_portal({"scheme": "invalid_scheme", "path": "/", "port": 8080}) + + +def test_add_portal_with_invalid_path(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.portals.add_portal({"scheme": "http", "path": "invalid_path", "port": 8080}) + + +def test_add_portal_with_invalid_path_double_slash(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.portals.add_portal({"scheme": "http", "path": "/some//path", "port": 8080}) + + +def test_add_portal_with_invalid_port(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.portals.add_portal({"scheme": "http", "path": "/", "port": -1}) diff --git a/library/2.0.31/tests/test_ports.py b/library/2.0.31/tests/test_ports.py new file mode 100644 index 0000000000..a4c923ca1d --- /dev/null +++ b/library/2.0.31/tests/test_ports.py @@ -0,0 +1,110 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_ports(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.ports.add_port(8081, 8080) + c1.ports.add_port(8082, 8080, {"protocol": "udp"}) + output = render.render() + assert output["services"]["test_container"]["ports"] == [ + {"published": 8081, "target": 8080, "protocol": "tcp", "mode": "ingress", "host_ip": "0.0.0.0"}, + {"published": 8082, "target": 8080, "protocol": "udp", "mode": "ingress", "host_ip": "0.0.0.0"}, + ] + + +def test_add_duplicate_ports(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.ports.add_port(8081, 8080) + c1.ports.add_port(8081, 8080, {"protocol": "udp"}) # This should not raise + with pytest.raises(Exception): + c1.ports.add_port(8081, 8080) + + +def test_add_duplicate_ports_with_different_host_ip(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.ports.add_port(8081, 8080, {"host_ip": "192.168.1.10"}) + c1.ports.add_port(8081, 8080, {"host_ip": "192.168.1.11"}) + output = render.render() + assert output["services"]["test_container"]["ports"] == [ + {"published": 8081, "target": 8080, "protocol": "tcp", "mode": "ingress", "host_ip": "192.168.1.10"}, + {"published": 8081, "target": 8080, "protocol": "tcp", "mode": "ingress", "host_ip": "192.168.1.11"}, + ] + + +def test_add_duplicate_ports_to_specific_host_ip_binds_to_0_0_0_0(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.ports.add_port(8081, 8080, {"host_ip": "192.168.1.10"}) + with pytest.raises(Exception): + c1.ports.add_port(8081, 8080, {"host_ip": "0.0.0.0"}) + + +def test_add_duplicate_ports_to_0_0_0_0_binds_to_specific_host_ip(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.ports.add_port(8081, 8080, {"host_ip": "0.0.0.0"}) + with pytest.raises(Exception): + c1.ports.add_port(8081, 8080, {"host_ip": "192.168.1.10"}) + + +def test_add_ports_with_invalid_protocol(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.ports.add_port(8081, 8080, {"protocol": "invalid_protocol"}) + + +def test_add_ports_with_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.ports.add_port(8081, 8080, {"mode": "invalid_mode"}) + + +def test_add_ports_with_invalid_ip(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.ports.add_port(8081, 8080, {"host_ip": "invalid_ip"}) + + +def test_add_ports_with_invalid_host_port(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.ports.add_port(-1, 8080) + + +def test_add_ports_with_invalid_container_port(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.ports.add_port(8081, -1) diff --git a/library/2.0.31/tests/test_render.py b/library/2.0.31/tests/test_render.py new file mode 100644 index 0000000000..60dc00679e --- /dev/null +++ b/library/2.0.31/tests/test_render.py @@ -0,0 +1,37 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_values_cannot_be_modified(mock_values): + render = Render(mock_values) + render.values["test"] = "test" + with pytest.raises(Exception): + render.render() + + +def test_duplicate_containers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_no_containers(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.render() diff --git a/library/2.0.31/tests/test_resources.py b/library/2.0.31/tests/test_resources.py new file mode 100644 index 0000000000..cd83d164e5 --- /dev/null +++ b/library/2.0.31/tests/test_resources.py @@ -0,0 +1,140 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_automatically_add_cpu(mock_values): + mock_values["resources"] = {"limits": {"cpus": 1.0}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["deploy"]["resources"]["limits"]["cpus"] == "1.0" + + +def test_invalid_cpu(mock_values): + mock_values["resources"] = {"limits": {"cpus": "invalid"}} + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_automatically_add_memory(mock_values): + mock_values["resources"] = {"limits": {"memory": 1024}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["deploy"]["resources"]["limits"]["memory"] == "1024M" + + +def test_invalid_memory(mock_values): + mock_values["resources"] = {"limits": {"memory": "invalid"}} + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_automatically_add_gpus(mock_values): + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + devices = output["services"]["test_container"]["deploy"]["resources"]["reservations"]["devices"] + assert len(devices) == 1 + assert devices[0] == { + "capabilities": ["gpu"], + "driver": "nvidia", + "device_ids": ["uuid_0", "uuid_1"], + } + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_gpu_without_uuid(mock_values): + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_remove_cpus_and_memory_with_gpus(mock_values): + mock_values["resources"] = {"gpus": {"nvidia_gpu_selection": {"pci_slot_0": {"uuid": "uuid_1", "use_gpu": True}}}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.deploy.resources.remove_cpus_and_memory() + output = render.render() + assert "limits" not in output["services"]["test_container"]["deploy"]["resources"] + devices = output["services"]["test_container"]["deploy"]["resources"]["reservations"]["devices"] + assert len(devices) == 1 + assert devices[0] == { + "capabilities": ["gpu"], + "driver": "nvidia", + "device_ids": ["uuid_1"], + } + + +def test_remove_cpus_and_memory(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.deploy.resources.remove_cpus_and_memory() + output = render.render() + assert "deploy" not in output["services"]["test_container"] + + +def test_remove_devices(mock_values): + mock_values["resources"] = {"gpus": {"nvidia_gpu_selection": {"pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}}}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.deploy.resources.remove_devices() + output = render.render() + assert "reservations" not in output["services"]["test_container"]["deploy"]["resources"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_set_profile(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.deploy.resources.set_profile("low") + output = render.render() + assert output["services"]["test_container"]["deploy"]["resources"]["limits"]["cpus"] == "1" + assert output["services"]["test_container"]["deploy"]["resources"]["limits"]["memory"] == "512M" + + +def test_set_profile_invalid_profile(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.deploy.resources.set_profile("invalid_profile") diff --git a/library/2.0.31/tests/test_restart.py b/library/2.0.31/tests/test_restart.py new file mode 100644 index 0000000000..06b2975590 --- /dev/null +++ b/library/2.0.31/tests/test_restart.py @@ -0,0 +1,57 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_invalid_restart_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.restart.set_policy("invalid_policy") + + +def test_valid_restart_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.restart.set_policy("on-failure") + output = render.render() + assert output["services"]["test_container"]["restart"] == "on-failure" + + +def test_valid_restart_policy_with_maximum_retry_count(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.restart.set_policy("on-failure", 10) + output = render.render() + assert output["services"]["test_container"]["restart"] == "on-failure:10" + + +def test_invalid_restart_policy_with_maximum_retry_count(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.restart.set_policy("on-failure", maximum_retry_count=-1) + + +def test_invalid_restart_policy_with_maximum_retry_count_and_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.restart.set_policy("always", maximum_retry_count=10) diff --git a/library/2.0.31/tests/test_sysctls.py b/library/2.0.31/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/library/2.0.31/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/library/2.0.31/tests/test_volumes.py b/library/2.0.31/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/library/2.0.31/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/library/2.0.31/validations.py b/library/2.0.31/validations.py new file mode 100644 index 0000000000..13f155dfdb --- /dev/null +++ b/library/2.0.31/validations.py @@ -0,0 +1,227 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/library/2.0.31/volume_mount.py b/library/2.0.31/volume_mount.py new file mode 100644 index 0000000000..aadd077750 --- /dev/null +++ b/library/2.0.31/volume_mount.py @@ -0,0 +1,92 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .formatter import merge_dicts_no_overwrite + from .volume_mount_types import BindMountType, VolumeMountType, TmpfsMountType + from .volume_sources import HostPathSource, IxVolumeSource, CifsSource, NfsSource, VolumeSource +except ImportError: + from error import RenderError + from formatter import merge_dicts_no_overwrite + from volume_mount_types import BindMountType, VolumeMountType, TmpfsMountType + from volume_sources import HostPathSource, IxVolumeSource, CifsSource, NfsSource, VolumeSource + + +class VolumeMount: + def __init__(self, render_instance: "Render", mount_path: str, config: "IxStorage"): + self._render_instance = render_instance + self.mount_path: str = mount_path + + storage_type: str = config.get("type", "") + if not storage_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match storage_type: + case "host_path": + spec_type = "bind" + mount_config = config.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + mount_type_specific_definition = BindMountType(self._render_instance, mount_config).render() + source = HostPathSource(self._render_instance, mount_config).get() + case "ix_volume": + spec_type = "bind" + mount_config = config.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + mount_type_specific_definition = BindMountType(self._render_instance, mount_config).render() + source = IxVolumeSource(self._render_instance, mount_config).get() + case "tmpfs": + spec_type = "tmpfs" + mount_config = config.get("tmpfs_config", {}) + mount_type_specific_definition = TmpfsMountType(self._render_instance, mount_config).render() + source = None + case "nfs": + spec_type = "volume" + mount_config = config.get("nfs_config") + if mount_config is None: + raise RenderError("Expected [nfs_config] to be set for [nfs] type.") + mount_type_specific_definition = VolumeMountType(self._render_instance, mount_config).render() + source = NfsSource(self._render_instance, mount_config).get() + case "cifs": + spec_type = "volume" + mount_config = config.get("cifs_config") + if mount_config is None: + raise RenderError("Expected [cifs_config] to be set for [cifs] type.") + mount_type_specific_definition = VolumeMountType(self._render_instance, mount_config).render() + source = CifsSource(self._render_instance, mount_config).get() + case "volume": + spec_type = "volume" + mount_config = config.get("volume_config") + if mount_config is None: + raise RenderError("Expected [volume_config] to be set for [volume] type.") + mount_type_specific_definition = VolumeMountType(self._render_instance, mount_config).render() + source = VolumeSource(self._render_instance, mount_config).get() + case "temporary": + spec_type = "volume" + mount_config = config.get("volume_config") + if mount_config is None: + raise RenderError("Expected [volume_config] to be set for [temporary] type.") + mount_type_specific_definition = VolumeMountType(self._render_instance, mount_config).render() + source = VolumeSource(self._render_instance, mount_config).get() + case "anonymous": + spec_type = "volume" + mount_config = config.get("volume_config") or {} + mount_type_specific_definition = VolumeMountType(self._render_instance, mount_config).render() + source = None + case _: + raise RenderError(f"Storage type [{storage_type}] is not supported for volume mounts.") + + common_spec = {"type": spec_type, "target": self.mount_path, "read_only": config.get("read_only", False)} + if source is not None: + common_spec["source"] = source + self._render_instance.volumes.add_volume(source, storage_type, mount_config) # type: ignore + + self.volume_mount_spec = merge_dicts_no_overwrite(common_spec, mount_type_specific_definition) + + def render(self) -> dict: + return self.volume_mount_spec diff --git a/library/2.0.31/volume_mount_types.py b/library/2.0.31/volume_mount_types.py new file mode 100644 index 0000000000..00a0ec3a18 --- /dev/null +++ b/library/2.0.31/volume_mount_types.py @@ -0,0 +1,72 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorageTmpfsConfig, IxStorageVolumeConfig, IxStorageBindLikeConfigs + + +try: + from .error import RenderError + from .validations import valid_host_path_propagation, valid_octal_mode_or_raise +except ImportError: + from error import RenderError + from validations import valid_host_path_propagation, valid_octal_mode_or_raise + + +class TmpfsMountType: + def __init__(self, render_instance: "Render", config: "IxStorageTmpfsConfig"): + self._render_instance = render_instance + self.spec = {"tmpfs": {}} + size = config.get("size", None) + mode = config.get("mode", None) + + if size is not None: + if not isinstance(size, int): + raise RenderError(f"Expected [size] to be an integer for [tmpfs] type, got [{size}]") + if not size > 0: + raise RenderError(f"Expected [size] to be greater than 0 for [tmpfs] type, got [{size}]") + # Convert Mebibytes to Bytes + self.spec["tmpfs"]["size"] = size * 1024 * 1024 + + if mode is not None: + mode = valid_octal_mode_or_raise(mode) + self.spec["tmpfs"]["mode"] = int(mode, 8) + + if not self.spec["tmpfs"]: + self.spec.pop("tmpfs") + + def render(self) -> dict: + """Render the tmpfs mount specification.""" + return self.spec + + +class BindMountType: + def __init__(self, render_instance: "Render", config: "IxStorageBindLikeConfigs"): + self._render_instance = render_instance + self.spec: dict = {} + + propagation = valid_host_path_propagation(config.get("propagation", "rprivate")) + create_host_path = config.get("create_host_path", False) + + self.spec: dict = { + "bind": { + "create_host_path": create_host_path, + "propagation": propagation, + } + } + + def render(self) -> dict: + """Render the bind mount specification.""" + return self.spec + + +class VolumeMountType: + def __init__(self, render_instance: "Render", config: "IxStorageVolumeConfig"): + self._render_instance = render_instance + self.spec: dict = {} + + self.spec: dict = {"volume": {"nocopy": config.get("nocopy", False)}} + + def render(self) -> dict: + """Render the volume mount specification.""" + return self.spec diff --git a/library/2.0.31/volume_sources.py b/library/2.0.31/volume_sources.py new file mode 100644 index 0000000000..c33fe55ea1 --- /dev/null +++ b/library/2.0.31/volume_sources.py @@ -0,0 +1,106 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorageHostPathConfig, IxStorageIxVolumeConfig, IxStorageVolumeConfig + +try: + from .error import RenderError + from .formatter import get_hashed_name_for_volume + from .validations import valid_fs_path_or_raise +except ImportError: + from error import RenderError + from formatter import get_hashed_name_for_volume + from validations import valid_fs_path_or_raise + + +class HostPathSource: + def __init__(self, render_instance: "Render", config: "IxStorageHostPathConfig"): + self._render_instance = render_instance + self.source: str = "" + + if not config: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + + path = "" + if config.get("acl_enable", False): + acl_path = config.get("acl", {}).get("path") + if not acl_path: + raise RenderError("Expected [host_path_config.acl.path] to be set for [host_path] type.") + path = valid_fs_path_or_raise(acl_path) + else: + path = valid_fs_path_or_raise(config.get("path", "")) + + self.source = path.rstrip("/") + + def get(self): + return self.source + + +class IxVolumeSource: + def __init__(self, render_instance: "Render", config: "IxStorageIxVolumeConfig"): + self._render_instance = render_instance + self.source: str = "" + + if not config: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + dataset_name = config.get("dataset_name") + if not dataset_name: + raise RenderError("Expected [ix_volume_config.dataset_name] to be set for [ix_volume] type.") + + ix_volumes = self._render_instance.values.get("ix_volumes", {}) + if dataset_name not in ix_volumes: + available = ", ".join(ix_volumes.keys()) + raise RenderError( + f"Expected the key [{dataset_name}] to be set in [ix_volumes] for [ix_volume] type. " + f"Available keys: [{available}]." + ) + + self.source = valid_fs_path_or_raise(ix_volumes[dataset_name].rstrip("/")) + + def get(self): + return self.source + + +class CifsSource: + def __init__(self, render_instance: "Render", config: dict): + self._render_instance = render_instance + self.source: str = "" + + if not config: + raise RenderError("Expected [cifs_config] to be set for [cifs] type.") + self.source = get_hashed_name_for_volume("cifs", config) + + def get(self): + return self.source + + +class NfsSource: + def __init__(self, render_instance: "Render", config: dict): + self._render_instance = render_instance + self.source: str = "" + + if not config: + raise RenderError("Expected [nfs_config] to be set for [nfs] type.") + self.source = get_hashed_name_for_volume("nfs", config) + + def get(self): + return self.source + + +class VolumeSource: + def __init__(self, render_instance: "Render", config: "IxStorageVolumeConfig"): + self._render_instance = render_instance + self.source: str = "" + + if not config: + raise RenderError("Expected [volume_config] to be set for [volume] type.") + + volume_name: str = config.get("volume_name", "") + if not volume_name: + raise RenderError("Expected [volume_config.volume_name] to be set for [volume] type.") + + self.source = volume_name + + def get(self): + return self.source diff --git a/library/2.0.31/volume_types.py b/library/2.0.31/volume_types.py new file mode 100644 index 0000000000..4ccea08f83 --- /dev/null +++ b/library/2.0.31/volume_types.py @@ -0,0 +1,133 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorageNfsConfig, IxStorageCifsConfig, IxStorageVolumeConfig + + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_fs_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_fs_path_or_raise + + +class NfsVolume: + def __init__(self, render_instance: "Render", config: "IxStorageNfsConfig"): + self._render_instance = render_instance + + if not config: + raise RenderError("Expected [nfs_config] to be set for [nfs] type") + + required_keys = ["server", "path"] + for key in required_keys: + if not config.get(key): + raise RenderError(f"Expected [{key}] to be set for [nfs] type") + + opts = [f"addr={config['server']}"] + cfg_options = config.get("options") + if cfg_options: + if not isinstance(cfg_options, list): + raise RenderError("Expected [nfs_config.options] to be a list for [nfs] type") + + tracked_keys: set[str] = set() + disallowed_opts = ["addr"] + for opt in cfg_options: + if not isinstance(opt, str): + raise RenderError("Options for [nfs] type must be a list of strings.") + + key = opt.split("=")[0] + if key in tracked_keys: + raise RenderError(f"Option [{key}] already added for [nfs] type.") + if key in disallowed_opts: + raise RenderError(f"Option [{key}] is not allowed for [nfs] type.") + opts.append(opt) + tracked_keys.add(key) + + opts.sort() + + path = valid_fs_path_or_raise(config["path"].rstrip("/")) + self.volume_spec = { + "driver_opts": { + "type": "nfs", + "device": f":{path}", + "o": f"{','.join([escape_dollar(opt) for opt in opts])}", + }, + } + + def get(self): + return self.volume_spec + + +class CifsVolume: + def __init__(self, render_instance: "Render", config: "IxStorageCifsConfig"): + self._render_instance = render_instance + self.volume_spec: dict = {} + + if not config: + raise RenderError("Expected [cifs_config] to be set for [cifs] type") + + required_keys = ["server", "path", "username", "password"] + for key in required_keys: + if not config.get(key): + raise RenderError(f"Expected [{key}] to be set for [cifs] type") + + opts = [ + "noperm", + f"user={config['username']}", + f"password={config['password']}", + ] + + domain = config.get("domain") + if domain: + opts.append(f"domain={domain}") + + cfg_options = config.get("options") + if cfg_options: + if not isinstance(cfg_options, list): + raise RenderError("Expected [cifs_config.options] to be a list for [cifs] type") + + tracked_keys: set[str] = set() + disallowed_opts = ["user", "password", "domain", "noperm"] + for opt in cfg_options: + if not isinstance(opt, str): + raise RenderError("Options for [cifs] type must be a list of strings.") + + key = opt.split("=")[0] + if key in tracked_keys: + raise RenderError(f"Option [{key}] already added for [cifs] type.") + if key in disallowed_opts: + raise RenderError(f"Option [{key}] is not allowed for [cifs] type.") + for disallowed in disallowed_opts: + if key == disallowed: + raise RenderError(f"Option [{key}] is not allowed for [cifs] type.") + opts.append(opt) + tracked_keys.add(key) + opts.sort() + + server = config["server"].lstrip("/") + path = config["path"].strip("/") + path = valid_fs_path_or_raise("/" + path).lstrip("/") + + self.volume_spec = { + "driver_opts": { + "type": "cifs", + "device": f"//{server}/{path}", + "o": f"{','.join([escape_dollar(opt) for opt in opts])}", + }, + } + + def get(self): + return self.volume_spec + + +class DockerVolume: + def __init__(self, render_instance: "Render", config: "IxStorageVolumeConfig"): + self._render_instance = render_instance + self.volume_spec: dict = {} + + def get(self): + return self.volume_spec diff --git a/library/2.0.31/volumes.py b/library/2.0.31/volumes.py new file mode 100644 index 0000000000..e6925a402f --- /dev/null +++ b/library/2.0.31/volumes.py @@ -0,0 +1,66 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + + +try: + from .error import RenderError + from .storage import IxStorageVolumeLikeConfigs + from .volume_types import NfsVolume, CifsVolume, DockerVolume +except ImportError: + from error import RenderError + from storage import IxStorageVolumeLikeConfigs + from volume_types import NfsVolume, CifsVolume, DockerVolume + + +class Volumes: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volumes: dict[str, Volume] = {} + + def add_volume( + self, + source: str, + storage_type: str, + config: "IxStorageVolumeLikeConfigs", + ): + # This method can be called many times from the volume mounts + # Only add the volume if it is not already added, but dont raise an error + if source == "": + raise RenderError(f"Volume source [{source}] cannot be empty") + + if source in self._volumes: + return + + self._volumes[source] = Volume(self._render_instance, storage_type, config) + + def has_volumes(self) -> bool: + return bool(self._volumes) + + def render(self): + return {name: v.render() for name, v in sorted(self._volumes.items()) if v.render() is not None} + + +class Volume: + def __init__( + self, + render_instance: "Render", + storage_type: str, + config: "IxStorageVolumeLikeConfigs", + ): + self._render_instance = render_instance + self.volume_spec: dict | None = {} + + match storage_type: + case "nfs": + self.volume_spec = NfsVolume(self._render_instance, config).get() # type: ignore + case "cifs": + self.volume_spec = CifsVolume(self._render_instance, config).get() # type: ignore + case "volume" | "temporary": + self.volume_spec = DockerVolume(self._render_instance, config).get() # type: ignore + case _: + self.volume_spec = None + + def render(self): + return self.volume_spec diff --git a/library/hashes.yaml b/library/hashes.yaml index 35aa98a265..ac31086ad3 100644 --- a/library/hashes.yaml +++ b/library/hashes.yaml @@ -1,3 +1,3 @@ 0.0.1: f074617a82a86d2a6cc78a4c8a4296fc9d168e456f12713e50c696557b302133 1.1.7: d05e43e25b7dc1736be6cc1efa4b9255368aa346e3e7a4350a38440f29b73186 -2.0.30: a87750c752394f7bf0eb461678564a9ef99bc4e4541787ef25e7be1095a6f879 +2.0.31: 384dc07207a29a9c830b3c8bb65786d0b83a7fd093f0d7f375cdbbcbff397bf2 From 1946496d8a8948697713576f31dc32fd9287a57d Mon Sep 17 00:00:00 2001 From: Stavros Kois Date: Tue, 3 Dec 2024 14:47:23 +0200 Subject: [PATCH 02/19] fix test and add labels --- ix-dev/community/whoogle/app.yaml | 4 +- ix-dev/community/whoogle/questions.yaml | 38 +++++++++++++++++++ .../__init__.py | 0 .../{base_v2_0_30 => base_v2_0_31}/configs.py | 0 .../container.py | 0 .../{base_v2_0_30 => base_v2_0_31}/depends.py | 0 .../{base_v2_0_30 => base_v2_0_31}/deploy.py | 0 .../{base_v2_0_30 => base_v2_0_31}/deps.py | 0 .../{base_v2_0_30 => base_v2_0_31}/device.py | 0 .../{base_v2_0_30 => base_v2_0_31}/devices.py | 0 .../{base_v2_0_30 => base_v2_0_31}/dns.py | 0 .../environment.py | 0 .../{base_v2_0_30 => base_v2_0_31}/error.py | 0 .../formatter.py | 0 .../functions.py | 0 .../healthcheck.py | 0 .../{base_v2_0_30 => base_v2_0_31}/labels.py | 0 .../{base_v2_0_30 => base_v2_0_31}/notes.py | 0 .../{base_v2_0_30 => base_v2_0_31}/portal.py | 0 .../{base_v2_0_30 => base_v2_0_31}/portals.py | 0 .../{base_v2_0_30 => base_v2_0_31}/ports.py | 0 .../{base_v2_0_30 => base_v2_0_31}/render.py | 0 .../resources.py | 0 .../{base_v2_0_30 => base_v2_0_31}/restart.py | 0 .../{base_v2_0_30 => base_v2_0_31}/storage.py | 0 .../{base_v2_0_30 => base_v2_0_31}/sysctls.py | 0 .../tests/__init__.py | 0 .../tests/test_build_image.py | 0 .../tests/test_configs.py | 0 .../tests/test_container.py | 0 .../tests/test_depends.py | 0 .../tests/test_deps.py | 0 .../tests/test_device.py | 0 .../tests/test_dns.py | 0 .../tests/test_environment.py | 0 .../tests/test_formatter.py | 0 .../tests/test_functions.py | 2 +- .../tests/test_healthcheck.py | 0 .../tests/test_labels.py | 0 .../tests/test_notes.py | 0 .../tests/test_portal.py | 0 .../tests/test_ports.py | 0 .../tests/test_render.py | 0 .../tests/test_resources.py | 0 .../tests/test_restart.py | 0 .../tests/test_sysctls.py | 0 .../tests/test_volumes.py | 0 .../validations.py | 0 .../volume_mount.py | 0 .../volume_mount_types.py | 0 .../volume_sources.py | 0 .../volume_types.py | 0 .../{base_v2_0_30 => base_v2_0_31}/volumes.py | 0 library/2.0.31/tests/test_functions.py | 2 +- library/hashes.yaml | 2 +- 55 files changed, 43 insertions(+), 5 deletions(-) rename ix-dev/community/whoogle/templates/library/{base_v2_0_30 => base_v2_0_31}/__init__.py (100%) rename ix-dev/community/whoogle/templates/library/{base_v2_0_30 => base_v2_0_31}/configs.py (100%) rename ix-dev/community/whoogle/templates/library/{base_v2_0_30 => base_v2_0_31}/container.py (100%) rename ix-dev/community/whoogle/templates/library/{base_v2_0_30 => base_v2_0_31}/depends.py (100%) rename ix-dev/community/whoogle/templates/library/{base_v2_0_30 => base_v2_0_31}/deploy.py (100%) rename ix-dev/community/whoogle/templates/library/{base_v2_0_30 => base_v2_0_31}/deps.py (100%) rename ix-dev/community/whoogle/templates/library/{base_v2_0_30 => base_v2_0_31}/device.py (100%) rename ix-dev/community/whoogle/templates/library/{base_v2_0_30 => base_v2_0_31}/devices.py (100%) rename ix-dev/community/whoogle/templates/library/{base_v2_0_30 => base_v2_0_31}/dns.py (100%) rename ix-dev/community/whoogle/templates/library/{base_v2_0_30 => base_v2_0_31}/environment.py (100%) rename ix-dev/community/whoogle/templates/library/{base_v2_0_30 => base_v2_0_31}/error.py (100%) rename ix-dev/community/whoogle/templates/library/{base_v2_0_30 => base_v2_0_31}/formatter.py (100%) rename ix-dev/community/whoogle/templates/library/{base_v2_0_30 => base_v2_0_31}/functions.py (100%) rename ix-dev/community/whoogle/templates/library/{base_v2_0_30 => base_v2_0_31}/healthcheck.py (100%) rename ix-dev/community/whoogle/templates/library/{base_v2_0_30 => base_v2_0_31}/labels.py (100%) rename ix-dev/community/whoogle/templates/library/{base_v2_0_30 => base_v2_0_31}/notes.py (100%) rename ix-dev/community/whoogle/templates/library/{base_v2_0_30 => base_v2_0_31}/portal.py (100%) rename ix-dev/community/whoogle/templates/library/{base_v2_0_30 => base_v2_0_31}/portals.py (100%) rename ix-dev/community/whoogle/templates/library/{base_v2_0_30 => base_v2_0_31}/ports.py (100%) rename ix-dev/community/whoogle/templates/library/{base_v2_0_30 => base_v2_0_31}/render.py (100%) rename ix-dev/community/whoogle/templates/library/{base_v2_0_30 => base_v2_0_31}/resources.py (100%) rename ix-dev/community/whoogle/templates/library/{base_v2_0_30 => base_v2_0_31}/restart.py (100%) rename ix-dev/community/whoogle/templates/library/{base_v2_0_30 => base_v2_0_31}/storage.py (100%) rename ix-dev/community/whoogle/templates/library/{base_v2_0_30 => base_v2_0_31}/sysctls.py (100%) rename ix-dev/community/whoogle/templates/library/{base_v2_0_30 => base_v2_0_31}/tests/__init__.py (100%) rename ix-dev/community/whoogle/templates/library/{base_v2_0_30 => base_v2_0_31}/tests/test_build_image.py (100%) rename ix-dev/community/whoogle/templates/library/{base_v2_0_30 => base_v2_0_31}/tests/test_configs.py (100%) rename ix-dev/community/whoogle/templates/library/{base_v2_0_30 => base_v2_0_31}/tests/test_container.py (100%) rename ix-dev/community/whoogle/templates/library/{base_v2_0_30 => base_v2_0_31}/tests/test_depends.py (100%) rename ix-dev/community/whoogle/templates/library/{base_v2_0_30 => base_v2_0_31}/tests/test_deps.py (100%) rename ix-dev/community/whoogle/templates/library/{base_v2_0_30 => base_v2_0_31}/tests/test_device.py (100%) rename ix-dev/community/whoogle/templates/library/{base_v2_0_30 => base_v2_0_31}/tests/test_dns.py (100%) rename ix-dev/community/whoogle/templates/library/{base_v2_0_30 => base_v2_0_31}/tests/test_environment.py (100%) rename ix-dev/community/whoogle/templates/library/{base_v2_0_30 => base_v2_0_31}/tests/test_formatter.py (100%) rename ix-dev/community/whoogle/templates/library/{base_v2_0_30 => base_v2_0_31}/tests/test_functions.py (98%) rename ix-dev/community/whoogle/templates/library/{base_v2_0_30 => base_v2_0_31}/tests/test_healthcheck.py (100%) rename ix-dev/community/whoogle/templates/library/{base_v2_0_30 => base_v2_0_31}/tests/test_labels.py (100%) rename ix-dev/community/whoogle/templates/library/{base_v2_0_30 => base_v2_0_31}/tests/test_notes.py (100%) rename ix-dev/community/whoogle/templates/library/{base_v2_0_30 => base_v2_0_31}/tests/test_portal.py (100%) rename ix-dev/community/whoogle/templates/library/{base_v2_0_30 => base_v2_0_31}/tests/test_ports.py (100%) rename ix-dev/community/whoogle/templates/library/{base_v2_0_30 => base_v2_0_31}/tests/test_render.py (100%) rename ix-dev/community/whoogle/templates/library/{base_v2_0_30 => base_v2_0_31}/tests/test_resources.py (100%) rename ix-dev/community/whoogle/templates/library/{base_v2_0_30 => base_v2_0_31}/tests/test_restart.py (100%) rename ix-dev/community/whoogle/templates/library/{base_v2_0_30 => base_v2_0_31}/tests/test_sysctls.py (100%) rename ix-dev/community/whoogle/templates/library/{base_v2_0_30 => base_v2_0_31}/tests/test_volumes.py (100%) rename ix-dev/community/whoogle/templates/library/{base_v2_0_30 => base_v2_0_31}/validations.py (100%) rename ix-dev/community/whoogle/templates/library/{base_v2_0_30 => base_v2_0_31}/volume_mount.py (100%) rename ix-dev/community/whoogle/templates/library/{base_v2_0_30 => base_v2_0_31}/volume_mount_types.py (100%) rename ix-dev/community/whoogle/templates/library/{base_v2_0_30 => base_v2_0_31}/volume_sources.py (100%) rename ix-dev/community/whoogle/templates/library/{base_v2_0_30 => base_v2_0_31}/volume_types.py (100%) rename ix-dev/community/whoogle/templates/library/{base_v2_0_30 => base_v2_0_31}/volumes.py (100%) diff --git a/ix-dev/community/whoogle/app.yaml b/ix-dev/community/whoogle/app.yaml index a529b2b092..1f39ce0f15 100644 --- a/ix-dev/community/whoogle/app.yaml +++ b/ix-dev/community/whoogle/app.yaml @@ -9,8 +9,8 @@ icon: https://media.sys.truenas.net/apps/whoogle/icons/icon.png keywords: - search - engine -lib_version: 2.0.30 -lib_version_hash: 384dc07207a29a9c830b3c8bb65786d0b83a7fd093f0d7f375cdbbcbff397bf2 +lib_version: 2.0.31 +lib_version_hash: 759e17c5d9384ac272d20fdd2cddef9d7a3e37192e4b0a354411e8a2a7e4ceb8 maintainers: - email: dev@ixsystems.com name: truenas diff --git a/ix-dev/community/whoogle/questions.yaml b/ix-dev/community/whoogle/questions.yaml index 27ebc855f5..f22d1f41e8 100644 --- a/ix-dev/community/whoogle/questions.yaml +++ b/ix-dev/community/whoogle/questions.yaml @@ -5,6 +5,8 @@ groups: description: Configure Network for Whoogle - name: Storage Configuration description: Configure Storage for Whoogle + - name: Labels Configuration + description: Configure Labels for Zerotier - name: Resources Configuration description: Configure Resources for Whoogle @@ -230,6 +232,42 @@ questions: description: The domain to use for the SMB share. schema: type: string + - variable: labels + label: "" + group: Labels Configuration + schema: + type: list + default: [] + items: + - variable: label + label: Label + schema: + type: dict + attrs: + - variable: key + label: Key + schema: + type: string + required: true + - variable: value + label: Value + schema: + type: string + required: true + - variable: containers + label: Containers + description: Containers where the label should be applied + schema: + type: list + items: + - variable: container + label: Container + schema: + type: string + required: true + enum: + - value: whoogle + description: whoogle - variable: resources label: "" group: Resources Configuration diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_30/__init__.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/__init__.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_30/__init__.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_31/__init__.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_30/configs.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/configs.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_30/configs.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_31/configs.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_30/container.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/container.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_30/container.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_31/container.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_30/depends.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/depends.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_30/depends.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_31/depends.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_30/deploy.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/deploy.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_30/deploy.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_31/deploy.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_30/deps.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/deps.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_30/deps.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_31/deps.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_30/device.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/device.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_30/device.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_31/device.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_30/devices.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/devices.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_30/devices.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_31/devices.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_30/dns.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/dns.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_30/dns.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_31/dns.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_30/environment.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/environment.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_30/environment.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_31/environment.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_30/error.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/error.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_30/error.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_31/error.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_30/formatter.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/formatter.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_30/formatter.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_31/formatter.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_30/functions.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/functions.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_30/functions.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_31/functions.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_30/healthcheck.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/healthcheck.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_30/healthcheck.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_31/healthcheck.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_30/labels.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/labels.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_30/labels.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_31/labels.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_30/notes.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/notes.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_30/notes.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_31/notes.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_30/portal.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/portal.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_30/portal.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_31/portal.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_30/portals.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/portals.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_30/portals.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_31/portals.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_30/ports.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/ports.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_30/ports.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_31/ports.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_30/render.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/render.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_30/render.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_31/render.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_30/resources.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/resources.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_30/resources.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_31/resources.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_30/restart.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/restart.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_30/restart.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_31/restart.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_30/storage.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/storage.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_30/storage.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_31/storage.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_30/sysctls.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/sysctls.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_30/sysctls.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_31/sysctls.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/__init__.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/__init__.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/__init__.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/__init__.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_build_image.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_build_image.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_build_image.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_build_image.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_configs.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_configs.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_configs.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_configs.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_container.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_container.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_container.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_container.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_depends.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_depends.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_depends.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_depends.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_deps.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_deps.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_deps.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_deps.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_device.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_device.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_device.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_device.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_dns.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_dns.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_dns.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_dns.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_environment.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_environment.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_environment.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_environment.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_formatter.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_formatter.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_formatter.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_formatter.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_functions.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_functions.py similarity index 98% rename from ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_functions.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_functions.py index c7e250fcf0..375c587ac8 100644 --- a/ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_functions.py +++ b/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_functions.py @@ -66,7 +66,7 @@ def test_funcs(mock_values): {"func": "or_default", "values": [1, None], "expected": 1}, {"func": "or_default", "values": [False, 1], "expected": 1}, {"func": "or_default", "values": [True, 1], "expected": True}, - {"func": "temp_config", "values": [""], "expected_raise": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, { "func": "temp_config", "values": ["test"], diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_healthcheck.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_healthcheck.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_healthcheck.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_healthcheck.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_labels.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_labels.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_labels.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_labels.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_notes.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_notes.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_notes.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_notes.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_portal.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_portal.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_portal.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_portal.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_ports.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_ports.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_ports.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_ports.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_render.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_render.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_render.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_render.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_resources.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_resources.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_resources.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_resources.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_restart.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_restart.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_restart.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_restart.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_sysctls.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_sysctls.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_sysctls.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_sysctls.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_volumes.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_volumes.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_30/tests/test_volumes.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_volumes.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_30/validations.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/validations.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_30/validations.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_31/validations.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_30/volume_mount.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/volume_mount.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_30/volume_mount.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_31/volume_mount.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_30/volume_mount_types.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/volume_mount_types.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_30/volume_mount_types.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_31/volume_mount_types.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_30/volume_sources.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/volume_sources.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_30/volume_sources.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_31/volume_sources.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_30/volume_types.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/volume_types.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_30/volume_types.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_31/volume_types.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_30/volumes.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/volumes.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_30/volumes.py rename to ix-dev/community/whoogle/templates/library/base_v2_0_31/volumes.py diff --git a/library/2.0.31/tests/test_functions.py b/library/2.0.31/tests/test_functions.py index c7e250fcf0..375c587ac8 100644 --- a/library/2.0.31/tests/test_functions.py +++ b/library/2.0.31/tests/test_functions.py @@ -66,7 +66,7 @@ def test_funcs(mock_values): {"func": "or_default", "values": [1, None], "expected": 1}, {"func": "or_default", "values": [False, 1], "expected": 1}, {"func": "or_default", "values": [True, 1], "expected": True}, - {"func": "temp_config", "values": [""], "expected_raise": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, { "func": "temp_config", "values": ["test"], diff --git a/library/hashes.yaml b/library/hashes.yaml index ac31086ad3..171a3e663e 100644 --- a/library/hashes.yaml +++ b/library/hashes.yaml @@ -1,3 +1,3 @@ 0.0.1: f074617a82a86d2a6cc78a4c8a4296fc9d168e456f12713e50c696557b302133 1.1.7: d05e43e25b7dc1736be6cc1efa4b9255368aa346e3e7a4350a38440f29b73186 -2.0.31: 384dc07207a29a9c830b3c8bb65786d0b83a7fd093f0d7f375cdbbcbff397bf2 +2.0.31: 759e17c5d9384ac272d20fdd2cddef9d7a3e37192e4b0a354411e8a2a7e4ceb8 From 4a78aa29fbe302be6c11c11ba332fe873296335e Mon Sep 17 00:00:00 2001 From: Stavros Kois Date: Tue, 3 Dec 2024 14:49:13 +0200 Subject: [PATCH 03/19] whoops --- ix-dev/community/whoogle/app.yaml | 2 +- .../templates/library/base_v2_0_31/tests/test_functions.py | 2 +- library/2.0.31/tests/test_functions.py | 2 +- library/hashes.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ix-dev/community/whoogle/app.yaml b/ix-dev/community/whoogle/app.yaml index 1f39ce0f15..b9dc31e058 100644 --- a/ix-dev/community/whoogle/app.yaml +++ b/ix-dev/community/whoogle/app.yaml @@ -10,7 +10,7 @@ keywords: - search - engine lib_version: 2.0.31 -lib_version_hash: 759e17c5d9384ac272d20fdd2cddef9d7a3e37192e4b0a354411e8a2a7e4ceb8 +lib_version_hash: e61b4db536830d207e591fea73037ca3f335da01a7e4073bb37392d1ede15873 maintainers: - email: dev@ixsystems.com name: truenas diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_functions.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_functions.py index 375c587ac8..0ea3b57d18 100644 --- a/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_functions.py +++ b/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_functions.py @@ -70,7 +70,7 @@ def test_funcs(mock_values): { "func": "temp_config", "values": ["test"], - "expected": {"type": "temporary", "volume_config": {"name": "test"}}, + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, }, ] diff --git a/library/2.0.31/tests/test_functions.py b/library/2.0.31/tests/test_functions.py index 375c587ac8..0ea3b57d18 100644 --- a/library/2.0.31/tests/test_functions.py +++ b/library/2.0.31/tests/test_functions.py @@ -70,7 +70,7 @@ def test_funcs(mock_values): { "func": "temp_config", "values": ["test"], - "expected": {"type": "temporary", "volume_config": {"name": "test"}}, + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, }, ] diff --git a/library/hashes.yaml b/library/hashes.yaml index 171a3e663e..80833e1d3f 100644 --- a/library/hashes.yaml +++ b/library/hashes.yaml @@ -1,3 +1,3 @@ 0.0.1: f074617a82a86d2a6cc78a4c8a4296fc9d168e456f12713e50c696557b302133 1.1.7: d05e43e25b7dc1736be6cc1efa4b9255368aa346e3e7a4350a38440f29b73186 -2.0.31: 759e17c5d9384ac272d20fdd2cddef9d7a3e37192e4b0a354411e8a2a7e4ceb8 +2.0.31: e61b4db536830d207e591fea73037ca3f335da01a7e4073bb37392d1ede15873 From db00e847e434877e8ae713f417fee03e2a67e186 Mon Sep 17 00:00:00 2001 From: Stavros Kois Date: Tue, 3 Dec 2024 14:45:30 +0200 Subject: [PATCH 04/19] init --- ix-dev/community/wordpress/app.yaml | 50 +++++++++---------- ix-dev/community/wordpress/ix_values.yaml | 2 - .../templates/test_values/basic-values.yaml | 18 ++++--- 3 files changed, 37 insertions(+), 33 deletions(-) diff --git a/ix-dev/community/wordpress/app.yaml b/ix-dev/community/wordpress/app.yaml index e55b95656b..fdcd27c575 100644 --- a/ix-dev/community/wordpress/app.yaml +++ b/ix-dev/community/wordpress/app.yaml @@ -1,41 +1,41 @@ app_version: 6.7.1 capabilities: -- description: Wordpress requires this ability to bind to port 80 within the container. - name: NET_BIND_SERVICE + - description: Wordpress requires this ability to bind to port 80 within the container. + name: NET_BIND_SERVICE categories: -- productivity + - productivity description: Wordpress is a web content management system home: https://wordpress.org host_mounts: [] icon: https://media.sys.truenas.net/apps/wordpress/icons/icon.png keywords: -- cms -- blog -lib_version: 1.1.7 + - cms + - blog +lib_version: 2.0.30 lib_version_hash: d05e43e25b7dc1736be6cc1efa4b9255368aa346e3e7a4350a38440f29b73186 maintainers: -- email: dev@ixsystems.com - name: truenas - url: https://www.truenas.com/ + - email: dev@ixsystems.com + name: truenas + url: https://www.truenas.com/ name: wordpress run_as_context: -- description: Wordpress runs as a non-root user. - gid: 33 - group_name: www-data - uid: 33 - user_name: www-data -- description: MariaDB runs as non-root user. - gid: 999 - group_name: mariadb - uid: 999 - user_name: mariadb + - description: Wordpress runs as a non-root user. + gid: 33 + group_name: www-data + uid: 33 + user_name: www-data + - description: MariaDB runs as non-root user. + gid: 999 + group_name: mariadb + uid: 999 + user_name: mariadb screenshots: -- https://media.sys.truenas.net/apps/wordpress/screenshots/screenshot1.png -- https://media.sys.truenas.net/apps/wordpress/screenshots/screenshot2.png -- https://media.sys.truenas.net/apps/wordpress/screenshots/screenshot3.png -- https://media.sys.truenas.net/apps/wordpress/screenshots/screenshot4.png + - https://media.sys.truenas.net/apps/wordpress/screenshots/screenshot1.png + - https://media.sys.truenas.net/apps/wordpress/screenshots/screenshot2.png + - https://media.sys.truenas.net/apps/wordpress/screenshots/screenshot3.png + - https://media.sys.truenas.net/apps/wordpress/screenshots/screenshot4.png sources: -- https://hub.docker.com/_/wordpress + - https://hub.docker.com/_/wordpress title: Wordpress train: community -version: 1.0.14 +version: 1.1.0 diff --git a/ix-dev/community/wordpress/ix_values.yaml b/ix-dev/community/wordpress/ix_values.yaml index 2de5e1dbc4..2cc59ba3ea 100644 --- a/ix-dev/community/wordpress/ix_values.yaml +++ b/ix-dev/community/wordpress/ix_values.yaml @@ -12,8 +12,6 @@ consts: mariadb_container_name: mariadb db_user: wordpress db_name: wordpress - mariadb_run_user: 999 - mariadb_run_group: 999 wordpress_run_user: 33 wordpress_run_group: 33 internal_web_port: 80 diff --git a/ix-dev/community/wordpress/templates/test_values/basic-values.yaml b/ix-dev/community/wordpress/templates/test_values/basic-values.yaml index 771bc038f3..2b12cc3a02 100644 --- a/ix-dev/community/wordpress/templates/test_values/basic-values.yaml +++ b/ix-dev/community/wordpress/templates/test_values/basic-values.yaml @@ -11,13 +11,19 @@ wordpress: network: web_port: 8080 +ix_volumes: + data: /opt/tests/mnt/data + mariadb_data: /opt/tests/mnt/mariadb_data + storage: data: - type: volume - volume_name: wordpress-data - auto_permissions: true + type: ix_volume + ix_volume_config: + dataset_name: data + create_host_path: true mariadb_data: - type: volume - auto_permissions: true - volume_name: wordpress-mariadb-data + type: ix_volume + ix_volume_config: + dataset_name: mariadb_data + create_host_path: true additional_storage: [] From 520c268ff9a3c67d4a2da085246646ccdb9b3a88 Mon Sep 17 00:00:00 2001 From: Stavros Kois Date: Tue, 3 Dec 2024 14:48:51 +0200 Subject: [PATCH 05/19] m --- ix-dev/community/wordpress/questions.yaml | 40 +++++++++++++++++++ .../wordpress/templates/docker-compose.yaml | 24 +++++------ 2 files changed, 52 insertions(+), 12 deletions(-) diff --git a/ix-dev/community/wordpress/questions.yaml b/ix-dev/community/wordpress/questions.yaml index 2347b221a4..6cde7c56b1 100644 --- a/ix-dev/community/wordpress/questions.yaml +++ b/ix-dev/community/wordpress/questions.yaml @@ -5,6 +5,8 @@ groups: description: Configure Network for Wordpress - name: Storage Configuration description: Configure Storage for Wordpress + - name: Labels Configuration + description: Configure Labels for Zerotier - name: Resources Configuration description: Configure Resources for Wordpress @@ -378,6 +380,44 @@ questions: description: The domain to use for the SMB share. schema: type: string + - variable: labels + label: "" + group: Labels Configuration + schema: + type: list + default: [] + items: + - variable: label + label: Label + schema: + type: dict + attrs: + - variable: key + label: Key + schema: + type: string + required: true + - variable: value + label: Value + schema: + type: string + required: true + - variable: containers + label: Containers + description: Containers where the label should be applied + schema: + type: list + items: + - variable: container + label: Container + schema: + type: string + required: true + enum: + - value: wordpress + description: wordpress + - value: mariadb + description: mariadb - variable: resources label: "" group: Resources Configuration diff --git a/ix-dev/community/wordpress/templates/docker-compose.yaml b/ix-dev/community/wordpress/templates/docker-compose.yaml index 78de21f134..5712ec3857 100644 --- a/ix-dev/community/wordpress/templates/docker-compose.yaml +++ b/ix-dev/community/wordpress/templates/docker-compose.yaml @@ -1,15 +1,15 @@ -{# Stores storage items that contains info for volumes, vol mounts, perms dirs and perms mounts #} -{% set storage_items = namespace(items=[]) %} -{% set mariadb_storage_items = namespace(items=[]) %} -{# Stores the top level volumes #} -{% set volumes = namespace(items={}) %} -{# Stores the container volume mounts #} -{% set volume_mounts = namespace(items=[]) %} -{% set mariadb_volume_mounts = namespace(items=[]) %} -{# Stores the perms container volume mounts #} -{% set perms_mounts = namespace(items=[]) %} -{# Stores the perms container dirs #} -{% set perms_dirs = namespace(items=[]) %} +{% set tpl = ix_lib.base.render.Render(values) %} + +{% set perm_container = tpl.deps.perms(values.consts.perms_container_name) %} +{% set mariadb_config = { + "user": values.consts.db_user, + "root_password": values.castopod.db_root_password, + "password": values.castopod.db_password, + "database": values.consts.db_name, + "volume": values.storage.mariadb_data, +} %} +{% set mariadb_container = tpl.deps.mariadb(values.consts.mariadb_container_name, "mariadb_image", mariadb_config, perm_container) %} + {% do storage_items.items.append(ix_lib.base.storage.storage_item(data=dict(values.storage.data, **{"mount_path": "/var/www/html"}), values=values, perm_opts={"mount_path": "/mnt/wordpress/html", "mode": "check", "uid": values.consts.wordpress_run_user, "gid": values.consts.wordpress_run_group} From bbe7e4529ad6d498e83621e77a9ce97bf348c85f Mon Sep 17 00:00:00 2001 From: Stavros Kois Date: Tue, 3 Dec 2024 14:55:24 +0200 Subject: [PATCH 06/19] wordpress: use v2 --- ix-dev/community/wordpress/app.yaml | 4 +- .../wordpress/templates/docker-compose.yaml | 127 ++---- .../templates/library/base_v1_1_7/__init__.py | 0 .../library/base_v1_1_7/environment.py | 98 ----- .../library/base_v1_1_7/healthchecks.py | 120 ------ .../templates/library/base_v1_1_7/mariadb.py | 72 ---- .../templates/library/base_v1_1_7/metadata.py | 71 ---- .../templates/library/base_v1_1_7/network.py | 21 - .../library/base_v1_1_7/permissions.py | 139 ------- .../templates/library/base_v1_1_7/ports.py | 42 -- .../templates/library/base_v1_1_7/postgres.py | 77 ---- .../templates/library/base_v1_1_7/redis.py | 49 --- .../library/base_v1_1_7/resources.py | 101 ----- .../templates/library/base_v1_1_7/security.py | 34 -- .../templates/library/base_v1_1_7/storage.py | 370 ------------------ .../templates/library/base_v1_1_7/utils.py | 124 ------ 16 files changed, 34 insertions(+), 1415 deletions(-) delete mode 100644 ix-dev/community/wordpress/templates/library/base_v1_1_7/__init__.py delete mode 100644 ix-dev/community/wordpress/templates/library/base_v1_1_7/environment.py delete mode 100644 ix-dev/community/wordpress/templates/library/base_v1_1_7/healthchecks.py delete mode 100644 ix-dev/community/wordpress/templates/library/base_v1_1_7/mariadb.py delete mode 100644 ix-dev/community/wordpress/templates/library/base_v1_1_7/metadata.py delete mode 100644 ix-dev/community/wordpress/templates/library/base_v1_1_7/network.py delete mode 100644 ix-dev/community/wordpress/templates/library/base_v1_1_7/permissions.py delete mode 100644 ix-dev/community/wordpress/templates/library/base_v1_1_7/ports.py delete mode 100644 ix-dev/community/wordpress/templates/library/base_v1_1_7/postgres.py delete mode 100644 ix-dev/community/wordpress/templates/library/base_v1_1_7/redis.py delete mode 100644 ix-dev/community/wordpress/templates/library/base_v1_1_7/resources.py delete mode 100644 ix-dev/community/wordpress/templates/library/base_v1_1_7/security.py delete mode 100644 ix-dev/community/wordpress/templates/library/base_v1_1_7/storage.py delete mode 100644 ix-dev/community/wordpress/templates/library/base_v1_1_7/utils.py diff --git a/ix-dev/community/wordpress/app.yaml b/ix-dev/community/wordpress/app.yaml index fdcd27c575..27205dfb88 100644 --- a/ix-dev/community/wordpress/app.yaml +++ b/ix-dev/community/wordpress/app.yaml @@ -11,8 +11,8 @@ icon: https://media.sys.truenas.net/apps/wordpress/icons/icon.png keywords: - cms - blog -lib_version: 2.0.30 -lib_version_hash: d05e43e25b7dc1736be6cc1efa4b9255368aa346e3e7a4350a38440f29b73186 +lib_version: 2.0.31 +lib_version_hash: a87750c752394f7bf0eb461678564a9ef99bc4e4541787ef25e7be1095a6f879 maintainers: - email: dev@ixsystems.com name: truenas diff --git a/ix-dev/community/wordpress/templates/docker-compose.yaml b/ix-dev/community/wordpress/templates/docker-compose.yaml index 5712ec3857..09a5b0c9d8 100644 --- a/ix-dev/community/wordpress/templates/docker-compose.yaml +++ b/ix-dev/community/wordpress/templates/docker-compose.yaml @@ -3,110 +3,47 @@ {% set perm_container = tpl.deps.perms(values.consts.perms_container_name) %} {% set mariadb_config = { "user": values.consts.db_user, - "root_password": values.castopod.db_root_password, - "password": values.castopod.db_password, + "root_password": values.wordpress.db_root_password, + "password": values.wordpress.db_password, "database": values.consts.db_name, "volume": values.storage.mariadb_data, } %} {% set mariadb_container = tpl.deps.mariadb(values.consts.mariadb_container_name, "mariadb_image", mariadb_config, perm_container) %} +{% set perms_config = {"uid": values.consts.wordpress_run_user, "gid": values.consts.wordpress_run_group, "mode": "check"} %} + +{% set c1 = tpl.add_container(values.consts.wordpress_container_name, "image") %} +{% do c1.set_user(values.consts.wordpress_run_user, values.consts.wordpress_run_group) %} +{% do c1.add_caps(["NET_BIND_SERVICE"]) %} +{% do c1.depends.add_dependency(values.consts.mariadb_container_name, "service_healthy") %} +{% do c1.healthcheck.set_test("tcp", {"port": values.consts.internal_web_port}) %} +{% do c1.environment.add_env("WORDPRESS_DB_HOST", values.consts.mariadb_container_name) %} +{% do c1.environment.add_env("WORDPRESS_DB_USER", values.consts.db_user) %} +{% do c1.environment.add_env("WORDPRESS_DB_PASSWORD", values.wordpress.db_password) %} +{% do c1.environment.add_env("WORDPRESS_DB_NAME", values.consts.db_name) %} +{% do c1.environment.add_user_envs(values.wordpress.additional_envs) %} + +{% if not values.network.host_network %} + {% do c1.ports.add_port(values.network.web_port, values.consts.internal_web_port) %} +{% endif %} +{% do c1.add_storage("/var/www/html", values.storage.data) %} +{% do perm_container.add_or_skip_action("data", values.storage.data, perms_config) %} -{% do storage_items.items.append(ix_lib.base.storage.storage_item(data=dict(values.storage.data, **{"mount_path": "/var/www/html"}), - values=values, perm_opts={"mount_path": "/mnt/wordpress/html", "mode": "check", "uid": values.consts.wordpress_run_user, "gid": values.consts.wordpress_run_group} -)) %} -{% do storage_items.items.append(ix_lib.base.storage.storage_item(data={"type":"temporary", "mount_path": "/tmp"}, - values=values, perm_opts={"mount_path": "/mnt/wordpress/tmp", "mode": "check", "uid": values.consts.wordpress_run_user, "gid": values.consts.wordpress_run_group} -)) %} -{% do storage_items.items.append(ix_lib.base.storage.storage_item(data={"type":"temporary", "mount_path": "/var/run"}, - values=values, perm_opts={"mount_path": "/mnt/wordpress/varrun", "mode": "check", "uid": values.consts.wordpress_run_user, "gid": values.consts.wordpress_run_group} -)) %} +{% do c1.add_storage("/var/run", tpl.funcs.temp_config("temp-var-run")) %} +{% do perm_container.add_or_skip_action("temp-var-run", tpl.funcs.temp_config("temp-var-run"), perms_config) %} {% for store in values.storage.additional_storage %} - {% do storage_items.items.append(ix_lib.base.storage.storage_item(data=store, values=values)) %} -{% endfor %} - -{# Add each item to the above lists #} -{% for item in storage_items.items %} - {% if item.vol and volumes.items.update(item.vol) %}{% endif %} - {% if item.vol_mount and volume_mounts.items.append(item.vol_mount) %}{% endif %} - {% if item.perms_item and (perms_dirs.items.append(item.perms_item.perm_dir), perms_mounts.items.append(item.perms_item.vol_mount)) %}{% endif %} -{% endfor %} - -{% do mariadb_storage_items.items.append(ix_lib.base.storage.storage_item(data={"type": "temporary", "mount_path": "/tmp"}, - perm_opts={"mount_path": "/mnt/mariadb/tmp", "mode": "check", "uid": values.consts.mariadb_run_user, "gid": values.consts.mariadb_run_group} -)) %} -{% do mariadb_storage_items.items.append(ix_lib.base.storage.storage_item(data=dict(values.storage.mariadb_data, **{"mount_path": "/var/lib/mysql"}), - values=values, perm_opts={"mount_path": "/mnt/mariadb/data", "mode": "check", "uid": values.consts.mariadb_run_user, "gid": values.consts.mariadb_run_group,} -)) %} -{% for item in mariadb_storage_items.items %} - {% if item.vol and volumes.items.update(item.vol) %}{% endif %} - {% if item.vol_mount and mariadb_volume_mounts.items.append(item.vol_mount) %}{% endif %} - {% if item.perms_item and (perms_dirs.items.append(item.perms_item.perm_dir), perms_mounts.items.append(item.perms_item.vol_mount)) %}{% endif %} + {% do c1.add_storage(store.mount_path, store) %} + {% do perm_container.add_or_skip_action(store.mount_path, store, perms_config) %} {% endfor %} -{# Containers #} -services: - {{ values.consts.wordpress_container_name }}: - user: {{ "%d:%d" | format(values.consts.wordpress_run_user, values.consts.wordpress_run_group) }} - image: {{ ix_lib.base.utils.get_image(images=values.images, name="image") }} - platform: linux/amd64 - restart: unless-stopped - deploy: - resources: {{ ix_lib.base.resources.resources(values.resources) | tojson }} - devices: {{ ix_lib.base.resources.get_devices(values.resources) | tojson }} - depends_on: - {{ values.consts.mariadb_container_name }}: - condition: service_healthy - {% if perms_dirs.items %} - {{ values.consts.perms_container_name }}: - condition: service_completed_successfully - {% endif %} - {% set caps = ix_lib.base.security.get_caps(add=["NET_BIND_SERVICE"]) %} - cap_add: {{ caps.add | tojson }} - cap_drop: {{ caps.drop | tojson }} - security_opt: {{ ix_lib.base.security.get_sec_opts() | tojson }} - {% if values.network.dns_opts %} - dns_opt: {{ ix_lib.base.network.dns_opts(values.network.dns_opts) | tojson }} - {% endif %} - {% set test = ix_lib.base.healthchecks.tcp_test(port=values.consts.internal_web_port) %} - healthcheck: {{ ix_lib.base.healthchecks.check_health(test) | tojson }} - environment: {{ ix_lib.base.environment.envs(app={ - "WORDPRESS_DB_HOST": values.consts.mariadb_container_name, - "WORDPRESS_DB_USER": values.consts.db_user, - "WORDPRESS_DB_PASSWORD": values.wordpress.db_password, - "WORDPRESS_DB_NAME": values.consts.db_name, - }, user=values.wordpress.additional_envs, values=values) | tojson }} - ports: - - {{ ix_lib.base.ports.get_port(port={"target": values.consts.internal_web_port, "published": values.network.web_port}) | tojson }} - volumes: {{ volume_mounts.items | tojson }} - - {% if perms_dirs.items %} - {{ values.consts.perms_container_name }}: {{ ix_lib.base.permissions.perms_container(items=perms_dirs.items, volumes=perms_mounts.items) | tojson }} - {% endif %} - - {% set resource_without_gpus = ix_lib.base.utils.copy_dict(values.resources) %} - {% do resource_without_gpus.pop("gpus", None) %} - {{ values.consts.mariadb_container_name }}: {{ ix_lib.base.mariadb.mariadb_container(data={ - "image": ix_lib.base.utils.get_image(images=values.images, name="mariadb_image"), - "volumes": mariadb_volume_mounts.items, - "user": values.consts.mariadb_run_user, "group": values.consts.mariadb_run_group, - "db_user": values.consts.db_user, "db_name": values.consts.db_name, - "db_password": values.wordpress.db_password, - "db_root_password": values.wordpress.db_root_password, - "dns_opts": values.network.dns_opts, "resources": resource_without_gpus, - "depends_on": { - values.consts.perms_container_name: { - "condition": "service_completed_successfully" - } if perms_dirs.items else {} - } - }) | tojson }} - -{% if volumes.items %} -volumes: {{ volumes.items | tojson }} +{% if perm_container.has_actions() %} + {% do perm_container.activate() %} + {% do c1.depends.add_dependency(values.consts.perms_container_name, "service_completed_successfully") %} + {% do mariadb_container.container.depends.add_dependency(values.consts.perms_container_name, "service_completed_successfully") %} {% endif %} -x-portals: {{ ix_lib.base.metadata.get_portals([ - {"port": values.network.web_port}, - {"name": "Admin Portal", "port": values.network.web_port, "path": "/wp-admin"}, -]) | tojson }} -x-notes: {{ ix_lib.base.metadata.get_notes("Wordpress") | tojson }} +{% do tpl.portals.add_portal({"port": values.network.web_port}) %} +{% do tpl.portals.add_portal({"name": "Admin Portal", "port": values.network.web_port, "path": "/wp-admin"},) %} + +{{ tpl.render() | tojson }} diff --git a/ix-dev/community/wordpress/templates/library/base_v1_1_7/__init__.py b/ix-dev/community/wordpress/templates/library/base_v1_1_7/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/ix-dev/community/wordpress/templates/library/base_v1_1_7/environment.py b/ix-dev/community/wordpress/templates/library/base_v1_1_7/environment.py deleted file mode 100644 index be5c8b7347..0000000000 --- a/ix-dev/community/wordpress/templates/library/base_v1_1_7/environment.py +++ /dev/null @@ -1,98 +0,0 @@ -from . import utils -from .resources import get_nvidia_gpus_reservations - - -def envs(app: dict | None = None, user: list | None = None, values: dict | None = None): - app = app or {} - user = user or [] - values = values or {} - result = {} - - if not values: - utils.throw_error("Values cannot be empty in environment.py") - - if not isinstance(user, list): - utils.throw_error( - f"Unsupported type for user environment variables [{type(user)}]" - ) - - # Always set TZ - result.update({"TZ": values.get("TZ", "Etc/UTC")}) - - # Update envs with nvidia variables - if values.get("resources", {}).get("gpus", {}): - result.update(get_nvidia_env(values.get("resources", {}).get("gpus", {}))) - - # Update envs with run_as variables - if values.get("run_as"): - result.update(get_run_as_envs(values.get("run_as", {}))) - - # Make sure we don't manually set any of the above - for item in app.items(): - if not item[0]: - utils.throw_error("Environment variable name cannot be empty.") - if item[0] in result: - utils.throw_error( - f"Environment variable [{item[0]}] is already defined automatically from the library." - ) - result[item[0]] = item[1] - - for item in user: - if not item.get("name"): - utils.throw_error("Environment variable name cannot be empty.") - if item.get("name") in result: - utils.throw_error( - f"Environment variable [{item['name']}] is already defined from the application developer." - ) - result[item["name"]] = item.get("value") - - for k, v in result.items(): - val = str(v) - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - val = val.lower() - result[k] = utils.escape_dollar(val) - - return result - - -# Sets some common variables that most applications use -def get_run_as_envs(run_as: dict) -> dict: - result = {} - user = run_as.get("user") - group = run_as.get("group") - if user: - result.update( - { - "PUID": user, - "UID": user, - "USER_ID": user, - } - ) - if group: - result.update( - { - "PGID": group, - "GID": group, - "GROUP_ID": group, - } - ) - return result - - -def get_nvidia_env(gpus: dict) -> dict: - reservations = get_nvidia_gpus_reservations(gpus) - if not reservations.get("device_ids"): - return { - "NVIDIA_VISIBLE_DEVICES": "void", - } - - return { - "NVIDIA_VISIBLE_DEVICES": ( - ",".join(reservations["device_ids"]) - if reservations.get("device_ids") - else "void" - ), - "NVIDIA_DRIVER_CAPABILITIES": "all", - } diff --git a/ix-dev/community/wordpress/templates/library/base_v1_1_7/healthchecks.py b/ix-dev/community/wordpress/templates/library/base_v1_1_7/healthchecks.py deleted file mode 100644 index cc98270d1d..0000000000 --- a/ix-dev/community/wordpress/templates/library/base_v1_1_7/healthchecks.py +++ /dev/null @@ -1,120 +0,0 @@ -from . import utils - - -def check_health(test, interval=10, timeout=5, retries=30, start_period=10): - if not test: - utils.throw_error("Expected [test] to be set") - - return { - "test": test, - "interval": f"{interval}s", - "timeout": f"{timeout}s", - "retries": retries, - "start_period": f"{start_period}s", - } - - -def mariadb_test(db, config=None): - config = config or {} - if not db: - utils.throw_error("MariaDB container: [db] must be set") - - host = config.get("host", "127.0.0.1") - port = config.get("port", 3306) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$$MARIADB_ROOT_PASSWORD ping" - - -def pg_test(user, db, config=None): - config = config or {} - if not user or not db: - utils.throw_error("Postgres container: [user] and [db] must be set") - - host = config.get("host", "127.0.0.1") - port = config.get("port", 5432) - - return f"pg_isready -h {host} -p {port} -d {db} -U {user}" - - -def redis_test(config=None): - config = config or {} - - host = config.get("host", "127.0.0.1") - port = config.get("port", 6379) - password = "$$REDIS_PASSWORD" - - return f"redis-cli -h {host} -p {port} -a {password} ping | grep -q PONG" - - -def curl_test(port, path, config=None): - config = config or {} - if not port or not path: - utils.throw_error("Expected [port] and [path] to be set") - - scheme = config.get("scheme", "http") - host = config.get("host", "127.0.0.1") - headers = config.get("headers", []) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - utils.throw_error("Expected [header] to be a list of two items") - opts.append(f'--header "{header[0]}: {header[1]}"') - - return f"curl --silent --output /dev/null --show-error --fail {' '.join(opts)} {scheme}://{host}:{port}{path}" - - -def wget_test(port, path, config=None): - config = config or {} - if not port or not path: - utils.throw_error("Expected [port] and [path] to be set") - - scheme = config.get("scheme", "http") - host = config.get("host", "127.0.0.1") - headers = config.get("headers", []) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - utils.throw_error("Expected [header] to be a list of two items") - opts.append(f'--header "{header[0]}: {header[1]}"') - - return f"wget --spider --quiet {' '.join(opts)} {scheme}://{host}:{port}{path}" - - -def http_test(port, path, config=None): - config = config or {} - if not port or not path: - utils.throw_error("Expected [port] and [path] to be set") - - host = config.get("host", "127.0.0.1") - - return f""" - /bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&$${{hc_fd}} && cat <&$${{hc_fd}} | grep "HTTP/1.1 200"' - """ # noqa - - -def netcat_test(port, config=None): - config = config or {} - if not port: - utils.throw_error("Expected [port] to be set") - - host = config.get("host", "127.0.0.1") - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(port, config=None): - config = config or {} - if not port: - utils.throw_error("Expected [port] to be set") - - host = config.get("host", "127.0.0.1") - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" diff --git a/ix-dev/community/wordpress/templates/library/base_v1_1_7/mariadb.py b/ix-dev/community/wordpress/templates/library/base_v1_1_7/mariadb.py deleted file mode 100644 index fc12daf81f..0000000000 --- a/ix-dev/community/wordpress/templates/library/base_v1_1_7/mariadb.py +++ /dev/null @@ -1,72 +0,0 @@ -from . import utils -from .security import get_caps, get_sec_opts -from .network import dns_opts -from .healthchecks import mariadb_test, check_health -from .resources import resources - - -def mariadb_env(user, password, root_password, dbname): - if not user: - utils.throw_error("Expected [user] to be set for mariadb") - if not password: - utils.throw_error("Expected [password] to be set for mariadb") - if not root_password: - utils.throw_error("Expected [root_password] to be set for mariadb") - if not dbname: - utils.throw_error("Expected [dbname] to be set for mariadb") - return { - "MARIADB_USER": user, - "MARIADB_PASSWORD": utils.escape_dollar(password), - "MARIADB_ROOT_PASSWORD": utils.escape_dollar(root_password), - "MARIADB_DATABASE": dbname, - "MARIADB_AUTO_UPGRADE": "true", - } - - -def mariadb_container(data={}): - req_keys = [ - "db_user", - "db_password", - "db_root_password", - "db_name", - "volumes", - "resources", - ] - for key in req_keys: - if not data.get(key): - utils.throw_error(f"Expected [{key}] to be set for mariadb") - - db_user = data["db_user"] - db_password = data["db_password"] - db_root_password = data["db_root_password"] - db_name = data["db_name"] - db_port = data.get("port", 3306) - depends = data.get("depends_on", {}) - depends_on = {} - for key in depends: - depends_on[key] = { - "condition": depends[key].get("condition", "service_completed_successfully") - } - - return { - "image": f"{data.get('image', 'mariadb:10.6')}", - "user": f"{data.get('user', '999')}:{data.get('group', '999')}", - "restart": "unless-stopped", - "cap_drop": get_caps()["drop"], - "security_opt": get_sec_opts(), - **({"dns_opts": dns_opts(data["dns_opts"])} if data.get("dns_opts") else {}), - "healthcheck": check_health(mariadb_test(db=db_name, config={"port": db_port})), - "command": [ - "--port", - str(db_port), - ], - "environment": mariadb_env( - user=db_user, - password=db_password, - root_password=db_root_password, - dbname=db_name, - ), - "volumes": data["volumes"], - "depends_on": depends_on, - "deploy": {"resources": resources(data["resources"])}, - } diff --git a/ix-dev/community/wordpress/templates/library/base_v1_1_7/metadata.py b/ix-dev/community/wordpress/templates/library/base_v1_1_7/metadata.py deleted file mode 100644 index c0a59f8979..0000000000 --- a/ix-dev/community/wordpress/templates/library/base_v1_1_7/metadata.py +++ /dev/null @@ -1,71 +0,0 @@ -from . import utils - - -def get_header(app_name: str): - return f"""# Welcome to TrueNAS SCALE - -Thank you for installing {app_name}! -""" - - -def get_footer(app_name: str): - return f"""## Documentation - -Documentation for {app_name} can be found at https://www.truenas.com/docs. - -## Bug reports - -If you find a bug in this app, please file an issue at -https://ixsystems.atlassian.net or https://github.com/truenas/apps - -## Feature requests or improvements - -If you find a feature request for this app, please file an issue at -https://ixsystems.atlassian.net or https://github.com/truenas/apps -""" - - -def get_notes(app_name: str, body: str = ""): - if not app_name: - utils.throw_error("Expected [app_name] to be set") - - return f"{get_header(app_name)}\n\n{body}\n\n{get_footer(app_name)}" - - -def get_portals(portals: list): - valid_schemes = ["http", "https"] - result = [] - for portal in portals: - # Most apps have a single portal, lets default to a standard name - name = portal.get("name", "Web UI") - scheme = portal.get("scheme", "http") - path = portal.get("path", "/") - - if not name: - utils.throw_error("Expected [portal.name] to be set") - if name in [p["name"] for p in result]: - utils.throw_error( - f"Expected [portal.name] to be unique, got [{', '.join([p['name'] for p in result]+[name])}]" - ) - if scheme not in valid_schemes: - utils.throw_error( - f"Expected [portal.scheme] to be one of [{', '.join(valid_schemes)}], got [{portal['scheme']}]" - ) - if not portal.get("port"): - utils.throw_error("Expected [portal.port] to be set") - if not path.startswith("/"): - utils.throw_error( - f"Expected [portal.path] to start with /, got [{portal['path']}]" - ) - - result.append( - { - "name": name, - "scheme": scheme, - "host": portal.get("host", "0.0.0.0"), - "port": portal["port"], - "path": path, - } - ) - - return result diff --git a/ix-dev/community/wordpress/templates/library/base_v1_1_7/network.py b/ix-dev/community/wordpress/templates/library/base_v1_1_7/network.py deleted file mode 100644 index e4761fd295..0000000000 --- a/ix-dev/community/wordpress/templates/library/base_v1_1_7/network.py +++ /dev/null @@ -1,21 +0,0 @@ -from . import utils - - -def dns_opts(dns_options=None): - dns_options = dns_options or [] - if not dns_options: - return [] - - tracked = {} - disallowed_opts = [] - for opt in dns_options: - key = opt.split(":")[0] - if key in tracked: - utils.throw_error( - f"Expected [dns_opts] to be unique, got [{', '.join([d.split(':')[0] for d in tracked])}]" - ) - if key in disallowed_opts: - utils.throw_error(f"Expected [dns_opts] to not contain [{key}] key.") - tracked[key] = opt - - return dns_options diff --git a/ix-dev/community/wordpress/templates/library/base_v1_1_7/permissions.py b/ix-dev/community/wordpress/templates/library/base_v1_1_7/permissions.py deleted file mode 100644 index 1ce6e60ca1..0000000000 --- a/ix-dev/community/wordpress/templates/library/base_v1_1_7/permissions.py +++ /dev/null @@ -1,139 +0,0 @@ -import jsonschema - -from . import utils - -ITEM_SCHEMA = { - "type": "object", - "properties": { - "dir": {"type": "string"}, - "mode": {"type": "string", "enum": ["always", "check"]}, - "uid": {"type": "integer"}, - "gid": {"type": "integer"}, - "chmod": {"type": "string"}, - "is_temporary": {"type": "boolean"}, - }, - "required": ["dir", "mode", "uid", "gid", "chmod", "is_temporary"], -} - - -def perms_container(items=[], volumes=[]): - if not items: - raise ValueError("Expected [items] to be set for perms_container") - if not volumes: - raise ValueError("Expected [volumes] to be set for perms_container") - - command = [process_dir_shell_func()] - for item in items: - try: - jsonschema.validate(item, ITEM_SCHEMA) - except jsonschema.ValidationError as e: - utils.throw_error(f"Item [{item}] is not valid: {e}") - cmd = [ - "process_dir", - item["dir"], - item["mode"], - str(item["uid"]), - str(item["gid"]), - item["chmod"], - str(item["is_temporary"]).lower(), - ] - command.append(" ".join(cmd)) - - return { - "image": "bash", - "user": "root", - "deploy": { - "resources": { - "limits": {"cpus": "1.0", "memory": "512m"}, - } - }, - "entrypoint": ["bash", "-c"], - "command": ["\n".join(command)], - "volumes": volumes, - } - - -# Don't forget to use double $ for shell variables, -# otherwise docker-compose will try to expand them -def process_dir_shell_func(): - return """ -function process_dir() { - local dir=$$1 - local mode=$$2 - local uid=$$3 - local gid=$$4 - local chmod=$$5 - local is_temporary=$$6 - - local fix_owner="false" - local fix_perms="false" - - if [ -z "$$dir" ]; then - echo "Path is empty, skipping..." - return 0 - fi - - if [ ! -d "$$dir" ]; then - echo "Path [$$dir] does is not a directory, skipping..." - return 0 - fi - - if [ "$$is_temporary" = "true" ]; then - echo "Path [$$dir] is a temporary directory, ensuring it is empty..." - # Exclude the safe directory, where we can use to mount files temporarily - find "$$dir" -mindepth 1 -maxdepth 1 ! -name "ix-safe" -exec rm -rf {} + - fi - - if [ "$$is_temporary" = "false" ] && [ -n "$$(ls -A $$dir)" ]; then - echo "Path [$$dir] is not empty, skipping..." - return 0 - fi - - echo "Current Ownership and Permissions on [$$dir]:" - echo "chown: $$(stat -c "%u %g" "$$dir")" - echo "chmod: $$(stat -c "%a" "$$dir")" - - if [ "$$mode" = "always" ]; then - fix_owner="true" - fix_perms="true" - fi - - if [ "$$mode" = "check" ]; then - if [ $$(stat -c %u "$$dir") -eq $$uid ] && [ $$(stat -c %g "$$dir") -eq $$gid ]; then - echo "Ownership is correct. Skipping..." - fix_owner="false" - else - echo "Ownership is incorrect. Fixing..." - fix_owner="true" - fi - - if [ "$$chmod" = "false" ]; then - echo "Skipping permissions check, chmod is false" - elif [ -n "$$chmod" ]; then - if [ $$(stat -c %a "$$dir") -eq $$chmod ]; then - echo "Permissions are correct. Skipping..." - fix_perms="false" - else - echo "Permissions are incorrect. Fixing..." - fix_perms="true" - fi - fi - fi - - if [ "$$fix_owner" = "true" ]; then - echo "Changing ownership to $$uid:$$gid on: [$$dir]" - chown -R "$$uid:$$gid" "$$dir" - echo "Finished changing ownership" - echo "Ownership after changes:" - stat -c "%u %g" "$$dir" - fi - - if [ -n "$$chmod" ] && [ "$$fix_perms" = "true" ]; then - echo "Changing permissions to $$chmod on: [$$dir]" - chmod -R "$$chmod" "$$dir" - echo "Finished changing permissions" - echo "Permissions after changes:" - stat -c "%a" "$$dir" - fi -} -""" diff --git a/ix-dev/community/wordpress/templates/library/base_v1_1_7/ports.py b/ix-dev/community/wordpress/templates/library/base_v1_1_7/ports.py deleted file mode 100644 index c895b47a44..0000000000 --- a/ix-dev/community/wordpress/templates/library/base_v1_1_7/ports.py +++ /dev/null @@ -1,42 +0,0 @@ -import ipaddress - -from . import utils - - -def must_valid_port(num: int): - if num < 1 or num > 65535: - utils.throw_error(f"Expected a valid port number, got [{num}]") - - -def must_valid_ip(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - utils.throw_error(f"Expected a valid IP address, got [{ip}]") - - -def must_valid_protocol(protocol: str): - if protocol not in ["tcp", "udp"]: - utils.throw_error(f"Expected a valid protocol, got [{protocol}]") - - -def must_valid_mode(mode: str): - if mode not in ["ingress", "host"]: - utils.throw_error(f"Expected a valid mode, got [{mode}]") - - -def get_port(port=None): - port = port or {} - must_valid_port(port["published"]) - must_valid_port(port["target"]) - must_valid_ip(port.get("host_ip", "0.0.0.0")) - must_valid_protocol(port.get("protocol", "tcp")) - must_valid_mode(port.get("mode", "ingress")) - - return { - "target": port["target"], - "published": port["published"], - "protocol": port.get("protocol", "tcp"), - "mode": port.get("mode", "ingress"), - "host_ip": port.get("host_ip", "0.0.0.0"), - } diff --git a/ix-dev/community/wordpress/templates/library/base_v1_1_7/postgres.py b/ix-dev/community/wordpress/templates/library/base_v1_1_7/postgres.py deleted file mode 100644 index c5f8275454..0000000000 --- a/ix-dev/community/wordpress/templates/library/base_v1_1_7/postgres.py +++ /dev/null @@ -1,77 +0,0 @@ -from . import utils -from .security import get_caps, get_sec_opts -from .network import dns_opts -from .healthchecks import pg_test, check_health -from .resources import resources - - -def pg_url(variant, host, user, password, dbname, port=5432): - if not host: - utils.throw_error("Expected [host] to be set") - if not user: - utils.throw_error("Expected [user] to be set") - if not password: - utils.throw_error("Expected [password] to be set") - if not dbname: - utils.throw_error("Expected [dbname] to be set") - - if variant == "postgresql": - return f"postgresql://{user}:{password}@{host}:{port}/{dbname}?sslmode=disable" - elif variant == "postgres": - return f"postgres://{user}:{password}@{host}:{port}/{dbname}?sslmode=disable" - else: - utils.throw_error( - f"Expected [variant] to be one of [postgresql, postgres], got [{variant}]" - ) - - -def pg_env(user, password, dbname, port=5432): - if not user: - utils.throw_error("Expected [user] to be set for postgres") - if not password: - utils.throw_error("Expected [password] to be set for postgres") - if not dbname: - utils.throw_error("Expected [dbname] to be set for postgres") - return { - "POSTGRES_USER": user, - "POSTGRES_PASSWORD": utils.escape_dollar(password), - "POSTGRES_DB": dbname, - "POSTGRES_PORT": port, - } - - -def pg_container(data={}): - req_keys = ["db_user", "db_password", "db_name", "volumes", "resources"] - for key in req_keys: - if not data.get(key): - utils.throw_error(f"Expected [{key}] to be set for postgres") - - pg_user = data["db_user"] - pg_password = data["db_password"] - pg_dbname = data["db_name"] - pg_port = data.get("port", 5432) - depends = data.get("depends_on", {}) - depends_on = {} - for key in depends: - depends_on[key] = { - "condition": depends[key].get("condition", "service_completed_successfully") - } - - return { - "image": f"{data.get('image', 'postgres:15')}", - "user": f"{data.get('user', '999')}:{data.get('group', '999')}", - "restart": "unless-stopped", - "cap_drop": get_caps()["drop"], - "security_opt": get_sec_opts(), - **({"dns_opts": dns_opts(data["dns_opts"])} if data.get("dns_opts") else {}), - "healthcheck": check_health(pg_test(user=pg_user, db=pg_dbname)), - "environment": pg_env( - user=pg_user, - password=pg_password, - dbname=pg_dbname, - port=pg_port, - ), - "volumes": data["volumes"], - "depends_on": depends_on, - "deploy": {"resources": resources(data["resources"])}, - } diff --git a/ix-dev/community/wordpress/templates/library/base_v1_1_7/redis.py b/ix-dev/community/wordpress/templates/library/base_v1_1_7/redis.py deleted file mode 100644 index 2356d0e06c..0000000000 --- a/ix-dev/community/wordpress/templates/library/base_v1_1_7/redis.py +++ /dev/null @@ -1,49 +0,0 @@ -from . import utils -from .security import get_caps, get_sec_opts -from .network import dns_opts -from .healthchecks import redis_test, check_health -from .resources import resources - - -def redis_container(data={}): - req_keys = ["password", "volumes", "resources"] - for key in req_keys: - if not data.get(key): - utils.throw_error(f"Expected [{key}] to be set for postgres") - - redis_password = data["password"] - redis_port = data.get("port", 6379) - depends = data.get("depends_on", {}) - depends_on = {} - for key in depends: - depends_on[key] = { - "condition": depends[key].get("condition", "service_completed_successfully") - } - - return { - "image": f"{data.get('image', 'bitnami/redis:7.0.11')}", - "user": f"{data.get('user', '1001')}:{data.get('group', '0')}", - "restart": "unless-stopped", - "cap_drop": get_caps()["drop"], - "security_opt": get_sec_opts(), - **({"dns_opts": dns_opts(data["dns_opts"])} if data.get("dns_opts") else {}), - "healthcheck": check_health(redis_test(config={"port": redis_port})), - "environment": redis_env( - password=redis_password, - port=redis_port, - ), - "volumes": data["volumes"], - "depends_on": depends_on, - "deploy": {"resources": resources(data["resources"])}, - } - - -def redis_env(password, port=6379): - if not password: - utils.throw_error("Expected [password] to be set for redis") - - return { - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": utils.escape_dollar(password), - "REDIS_PORT_NUMBER": port, - } diff --git a/ix-dev/community/wordpress/templates/library/base_v1_1_7/resources.py b/ix-dev/community/wordpress/templates/library/base_v1_1_7/resources.py deleted file mode 100644 index d3235fd565..0000000000 --- a/ix-dev/community/wordpress/templates/library/base_v1_1_7/resources.py +++ /dev/null @@ -1,101 +0,0 @@ -import re - -from . import utils - - -def resources(resources, disable_resource_limits=False): - gpus = resources.get("gpus", {}) - cpus = str(resources.get("limits", {}).get("cpus", 2.0)) - memory = str(resources.get("limits", {}).get("memory", 4096)) - if not re.match(r"^[1-9][0-9]*(\.[0-9]+)?$", cpus): - utils.throw_error(f"Expected cpus to be a number or a float, got [{cpus}]") - if not re.match(r"^[1-9][0-9]*$", memory): - raise ValueError(f"Expected memory to be a number, got [{memory}]") - - result = { - "limits": {"cpus": cpus, "memory": f"{memory}M"}, - "reservations": {"devices": []}, - } - - if gpus: - gpu_result = get_nvidia_gpus_reservations(gpus) - if gpu_result: - # Appending to devices, as we can later extend this to support other types of devices. Eg. TPUs. - result["reservations"]["devices"].append(get_nvidia_gpus_reservations(gpus)) - - # Docker does not like empty "things" all around. - if not result["reservations"]["devices"]: - del result["reservations"] - - if disable_resource_limits: - del result["limits"] - - return result - - -def get_nvidia_gpus_reservations(gpus: dict) -> dict: - """ - Input: - { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - """ - if not gpus: - return {} - - device_ids = [] - for pci, gpu in gpus.get("nvidia_gpu_selection", {}).items(): - if gpu["use_gpu"]: - if not gpu.get("uuid"): - utils.throw_error( - "Expected [uuid] to be set for GPU in" - f"slot [{pci}] in [nvidia_gpu_selection]" - ) - device_ids.append(gpu["uuid"]) - - if not device_ids: - return {} - - return { - "capabilities": ["gpu"], - "driver": "nvidia", - "device_ids": device_ids, - } - - -disallowed_devices = ["/dev/dri"] - - -# Returns the top level devices list -# Accepting other_devices to allow manually adding devices -# directly to the list. (Eg sound devices) -def get_devices(resources: dict, other_devices: list = []) -> list: - devices = [] - if resources.get("gpus", {}).get("use_all_gpus", False): - devices.append("/dev/dri:/dev/dri") - - added_host_devices: list = [] - for device in other_devices: - host_device = device.get("host_device", "").rstrip("/") - container_device = device.get("container_device", "") or host_device - if not host_device: - utils.throw_error(f"Expected [host_device] to be set for device [{device}]") - if not utils.valid_path(host_device): - utils.throw_error( - f"Expected [host_device] to be a valid path for device [{device}]" - ) - if host_device in disallowed_devices: - utils.throw_error( - f"Device [{host_device}] is not allowed to be manually added." - ) - if host_device in added_host_devices: - utils.throw_error( - f"Expected devices to be unique, but [{host_device}] was already added." - ) - devices.append(f"{host_device}:{container_device}") - added_host_devices.append(host_device) - - return devices diff --git a/ix-dev/community/wordpress/templates/library/base_v1_1_7/security.py b/ix-dev/community/wordpress/templates/library/base_v1_1_7/security.py deleted file mode 100644 index b67668b51b..0000000000 --- a/ix-dev/community/wordpress/templates/library/base_v1_1_7/security.py +++ /dev/null @@ -1,34 +0,0 @@ -from base64 import b64encode - -from . import utils - - -def get_caps(add=None, drop=None): - add = add or [] - drop = drop or ["ALL"] - result = {"drop": drop} - if add: - result["add"] = add - return result - - -def get_sec_opts(add=None, remove=None): - add = add or [] - remove = remove or [] - result = ["no-new-privileges"] - for opt in add: - if opt not in result: - result.append(opt) - for opt in remove: - if opt in result: - result.remove(opt) - return result - - -def htpasswd(username, password): - hashed = utils.bcrypt_hash(password) - return username + ":" + hashed - - -def basic_auth(username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") diff --git a/ix-dev/community/wordpress/templates/library/base_v1_1_7/storage.py b/ix-dev/community/wordpress/templates/library/base_v1_1_7/storage.py deleted file mode 100644 index de09ba0c3e..0000000000 --- a/ix-dev/community/wordpress/templates/library/base_v1_1_7/storage.py +++ /dev/null @@ -1,370 +0,0 @@ -import re -import json -import hashlib - -from . import utils - - -BIND_TYPES = ["host_path", "ix_volume"] -VOL_TYPES = ["volume", "nfs", "cifs", "temporary"] -ALL_TYPES = BIND_TYPES + VOL_TYPES + ["tmpfs", "anonymous"] -PROPAGATION_TYPES = ["shared", "slave", "private", "rshared", "rslave", "rprivate"] - - -def _get_name_for_temporary(data): - if not data.get("mount_path"): - utils.throw_error("Expected [mount_path] to be set for temporary volume") - - return data["mount_path"].lstrip("/").lower().replace("/", "_").replace(".", "_").replace(" ", "_") - - -# Returns a volume mount object (Used in container's "volumes" level) -def vol_mount(data, values=None): - values = values or {} - ix_volumes = values.get("ix_volumes") or [] - vol_type = _get_docker_vol_type(data) - - volume = { - "type": vol_type, - "target": utils.valid_path(data.get("mount_path", "")), - "read_only": data.get("read_only", False), - } - if vol_type == "bind": # Default create_host_path is true in short-syntax - volume.update(_get_bind_vol_config(data, values, ix_volumes)) - elif vol_type == "volume": - volume.update(_get_volume_vol_config(data)) - elif vol_type == "tmpfs": - volume.update(_get_tmpfs_vol_config(data)) - elif vol_type == "temporary": - volume["type"] = "volume" - volume.update(_get_volume_vol_config(data)) - elif vol_type == "anonymous": - volume["type"] = "volume" - volume.update(_get_anonymous_vol_config(data)) - - return volume - - -def storage_item(data, values=None, perm_opts=None): - values = values or {} - perm_opts = perm_opts or {} - if data.get("type") == "temporary": - data.update({"volume_name": _get_name_for_temporary(data)}) - return { - "vol_mount": vol_mount(data, values), - "vol": vol(data), - "perms_item": perms_item(data, values, perm_opts) if perm_opts else {}, - } - - -def perms_item(data, values=None, opts=None): - opts = opts or {} - values = values or {} - vol_type = data.get("type", "") - - # Temp volumes are always auto permissions - if vol_type == "temporary": - data.update({"auto_permissions": True}) - - # If its ix_volume, we need to set auto permissions - if vol_type == "ix_volume": - data.update({"auto_permissions": True}) - - if not data.get("auto_permissions"): - return {} - - if vol_type == "host_path": - if data.get("host_path_config", {}).get("acl_enable", False): - return {} - if vol_type == "ix_volume": - if data.get("ix_volume_config", {}).get("acl_enable", False): - return {} - - req_keys = ["mount_path", "mode", "uid", "gid"] - for key in req_keys: - if opts.get(key, None) is None: - utils.throw_error(f"Expected opts passed to [perms_item] to have [{key}] key") - - data.update({"mount_path": opts["mount_path"]}) - volume_mount = vol_mount(data, values) - # For perms volume mount, always set read_only to false - volume_mount.update({"read_only": False}) - - return { - "vol_mount": volume_mount, - "perm_dir": { - "dir": volume_mount["target"], - "mode": opts["mode"], - "uid": opts["uid"], - "gid": opts["gid"], - "chmod": opts.get("chmod", "false"), - "is_temporary": data["type"] == "temporary", - }, - } - - -def create_host_path_default(values): - """ - By default, do not create host path for bind mounts if it does not exist. - If the ix_context is missing, we are either in local dev or CI. - We should create the host path by default there to ease development. - The _magic_ "dev_mode" flag is added so we can also toggle this behavior - in CI, while we are also using ix_context for other tests. - """ - ix_ctx = values.get("ix_context", {}) - if not ix_ctx: - return True - if "dev_mode" in ix_ctx: - return ix_ctx["dev_mode"] - return False - - -def _get_bind_vol_config(data, values, ix_volumes=None): - ix_volumes = ix_volumes or [] - path = host_path(data, ix_volumes) - if data.get("propagation", "rprivate") not in PROPAGATION_TYPES: - utils.throw_error( - f"Expected [propagation] to be one of [{', '.join(PROPAGATION_TYPES)}], got [{data['propagation']}]" - ) - - # https://docs.docker.com/storage/bind-mounts/#configure-bind-propagation - return { - "source": path, - "bind": { - "create_host_path": data.get("host_path_config", {}).get( - "create_host_path", create_host_path_default(values) - ), - "propagation": _get_valid_propagation(data), - }, - } - - -def _get_volume_vol_config(data): - if data.get("type") in ["nfs", "cifs"]: - if data.get("volume_name"): - utils.throw_error("Expected [volume_name] to be empty for [nfs, cifs] type") - data.update({"volume_name": _get_name_for_external_volume(data)}) - if not data.get("volume_name"): - utils.throw_error("Expected [volume_name] to be set for [volume] type") - - return {"source": data["volume_name"], "volume": _process_volume_config(data)} - - -def _get_anonymous_vol_config(data): - return {"volume": _process_volume_config(data)} - - -mode_regex = re.compile(r"^0[0-7]{3}$") - - -def _get_tmpfs_vol_config(data): - tmpfs = {} - config = data.get("tmpfs_config", {}) - - if config.get("size"): - if not isinstance(config["size"], int): - utils.throw_error("Expected [size] to be an integer for [tmpfs] type") - if not config["size"] > 0: - utils.throw_error("Expected [size] to be greater than 0 for [tmpfs] type") - # Convert Mebibytes to Bytes - tmpfs.update({"size": config["size"] * 1024 * 1024}) - - if config.get("mode"): - if not mode_regex.match(str(config["mode"])): - utils.throw_error(f"Expected [mode] to be a octal string for [tmpfs] type, got [{config['mode']}]") - tmpfs.update({"mode": int(config["mode"], 8)}) - - return {"tmpfs": tmpfs} - - -# We generate a unique name for the volume based on the config -# Docker will not update any volume after creation. This is to ensure -# that changing any value (eg server address) in the config will result in a new volume -def _get_name_for_external_volume(data): - config_hash = hashlib.sha256(json.dumps(data).encode("utf-8")).hexdigest() - return f"{data['type']}_{config_hash}" - - -# Returns a volume object (Used in top "volumes" level) -def vol(data): - if not data or _get_docker_vol_type(data) != "volume": - return {} - - if not data.get("volume_name"): - utils.throw_error("Expected [volume_name] to be set for [volume] type") - - if data["type"] == "nfs": - return {data["volume_name"]: _process_nfs(data)} - elif data["type"] == "cifs": - return {data["volume_name"]: _process_cifs(data)} - else: - return {data["volume_name"]: {}} - - -def _is_host_path(data): - return data.get("type") == "host_path" - - -def _get_valid_propagation(data): - if not data.get("propagation"): - return "rprivate" - if not data["propagation"] in PROPAGATION_TYPES: - utils.throw_error( - f"Expected [propagation] to be one of [{', '.join(PROPAGATION_TYPES)}], got [{data['propagation']}]" - ) - return data["propagation"] - - -def _is_ix_volume(data): - return data.get("type") == "ix_volume" - - -# Returns the host path for a for either a host_path or ix_volume -def host_path(data, ix_volumes=None): - ix_volumes = ix_volumes or [] - path = "" - if _is_host_path(data): - path = _process_host_path_config(data) - elif _is_ix_volume(data): - path = _process_ix_volume_config(data, ix_volumes) - else: - utils.throw_error( - f"Expected [host_path()] to be called only for types [host_path, ix_volume], got [{data['type']}]" - ) - - return utils.valid_path(path) - - -# Returns the type of storage as used in docker-compose -def _get_docker_vol_type(data): - if not data.get("type"): - utils.throw_error("Expected [type] to be set for storage") - - if data["type"] not in ALL_TYPES: - utils.throw_error(f"Expected storage [type] to be one of {ALL_TYPES}, got [{data['type']}]") - - if data["type"] in BIND_TYPES: - return "bind" - elif data["type"] in VOL_TYPES: - return "volume" - else: - return data["type"] - - -def _process_host_path_config(data): - if data.get("host_path_config", {}).get("acl_enable", False): - if not data["host_path_config"].get("acl", {}).get("path"): - utils.throw_error("Expected [host_path_config.acl.path] to be set for [host_path] type with ACL enabled") - return data["host_path_config"]["acl"]["path"] - - if not data.get("host_path_config", {}).get("path"): - utils.throw_error("Expected [host_path_config.path] to be set for [host_path] type") - - return data["host_path_config"]["path"] - - -def _process_volume_config(data): - return {"nocopy": data.get("volume_config", {}).get("nocopy", False)} - - -def _process_ix_volume_config(data, ix_volumes): - path = "" - if not data.get("ix_volume_config", {}).get("dataset_name"): - utils.throw_error("Expected [ix_volume_config.dataset_name] to be set for [ix_volume] type") - - if not ix_volumes: - utils.throw_error("Expected [ix_volumes] to be set for [ix_volume] type") - - ds = data["ix_volume_config"]["dataset_name"] - path = ix_volumes.get(ds, None) - if not path: - utils.throw_error(f"Expected the key [{ds}] to be set in [ix_volumes]") - - return path - - -# Constructs a volume object for a cifs type -def _process_cifs(data): - if not data.get("cifs_config"): - utils.throw_error("Expected [cifs_config] to be set for [cifs] type") - - required_keys = ["server", "path", "username", "password"] - for key in required_keys: - if not data["cifs_config"].get(key): - utils.throw_error(f"Expected [{key}] to be set for [cifs] type") - - opts = [ - "noperm", - f"user={data['cifs_config']['username']}", - f"password={data['cifs_config']['password']}", - ] - if data["cifs_config"].get("domain"): - opts.append(f'domain={data["cifs_config"]["domain"]}') - - if data["cifs_config"].get("options"): - if not isinstance(data["cifs_config"]["options"], list): - utils.throw_error("Expected [cifs_config.options] to be a list for [cifs] type") - - disallowed_opts = ["user", "password", "domain", "noperm"] - for opt in data["cifs_config"]["options"]: - if not isinstance(opt, str): - utils.throw_error("Expected [cifs_config.options] to be a list of strings for [cifs] type") - - key = opt.split("=")[0] - for disallowed in disallowed_opts: - if key == disallowed: - utils.throw_error( - f"Expected [cifs_config.options] to not start with [{disallowed}] for [cifs] type" - ) - - opts.append(opt) - - server = data["cifs_config"]["server"].lstrip("/") - path = data["cifs_config"]["path"].strip("/") - volume = { - "driver_opts": { - "type": "cifs", - "device": f"//{server}/{path}", - "o": f"{','.join([utils.escape_dollar(opt) for opt in opts])}", - }, - } - - return volume - - -# Constructs a volume object for a nfs type -def _process_nfs(data): - if not data.get("nfs_config"): - utils.throw_error("Expected [nfs_config] to be set for [nfs] type") - - required_keys = ["server", "path"] - for key in required_keys: - if not data["nfs_config"].get(key): - utils.throw_error(f"Expected [{key}] to be set for [nfs] type") - - opts = [f"addr={data['nfs_config']['server']}"] - if data["nfs_config"].get("options"): - if not isinstance(data["nfs_config"]["options"], list): - utils.throw_error("Expected [nfs_config.options] to be a list for [nfs] type") - - disallowed_opts = ["addr"] - for opt in data["nfs_config"]["options"]: - if not isinstance(opt, str): - utils.throw_error("Expected [nfs_config.options] to be a list of strings for [nfs] type") - - key = opt.split("=")[0] - for disallowed in disallowed_opts: - if key == disallowed: - utils.throw_error(f"Expected [nfs_config.options] to not start with [{disallowed}] for [nfs] type") - - opts.append(opt) - - volume = { - "driver_opts": { - "type": "nfs", - "device": f":{data['nfs_config']['path']}", - "o": f"{','.join([utils.escape_dollar(opt) for opt in opts])}", - }, - } - - return volume diff --git a/ix-dev/community/wordpress/templates/library/base_v1_1_7/utils.py b/ix-dev/community/wordpress/templates/library/base_v1_1_7/utils.py deleted file mode 100644 index 8a7c0815c6..0000000000 --- a/ix-dev/community/wordpress/templates/library/base_v1_1_7/utils.py +++ /dev/null @@ -1,124 +0,0 @@ -import hashlib -import secrets -import bcrypt -import sys -import re - -from . import security - - -class TemplateException(Exception): - pass - - -def throw_error(message): - # When throwing a known error, hide the traceback - # This is because the error is also shown in the UI - # and having a traceback makes it hard for user to read - sys.tracebacklimit = 0 - raise TemplateException(message) - - -def secure_string(length): - return secrets.token_urlsafe(length) - - -def basic_auth_header(username, password): - return f"Basic {security.basic_auth(username, password)}" - - -def bcrypt_hash(password): - return bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - - -def match_regex(value, regex): - if not re.match(regex, value): - return False - return True - - -def must_match_regex(value, regex): - if not match_regex(value, regex): - throw_error(f"Expected [{value}] to match [{regex}]") - return value - - -def merge_dicts(*dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - -# Basic validation for a path (Expand later) -def valid_path(path=""): - if not path.startswith("/"): - throw_error(f"Expected path [{path}] to start with /") - - # There is no reason to allow / as a path, either on host or in a container - if path == "/": - throw_error(f"Expected path [{path}] to not be /") - - return path - - -def camel_case(string): - return string.title() - - -def is_boolean(string): - return string.lower() in ["true", "false"] - - -def is_number(string): - try: - float(string) - return True - except ValueError: - return False - - -def get_image(images={}, name=""): - if not images: - throw_error("Expected [images] to be set") - if name not in images: - throw_error(f"Expected [images.{name}] to be set") - if not images[name].get("repository") or not images[name].get("tag"): - throw_error(f"Expected [images.{name}.repository] and [images.{name}.tag] to be set") - - return f"{images[name]['repository']}:{images[name]['tag']}" - - -def hash_data(data=""): - if not data: - throw_error("Expected [data] to be set") - return hashlib.sha256(data.encode("utf-8")).hexdigest() - - -def get_image_with_hashed_data(images={}, name="", data=""): - return f"ix-{get_image(images, name)}-{hash_data(data)}" - - -def copy_dict(dict): - return dict.copy() - - -def escape_dollar(text: str) -> str: - return text.replace("$", "$$") - - -def auto_cast(value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value From c55d24e6b73b86f216d7c9167a0b95de5baaa8a9 Mon Sep 17 00:00:00 2001 From: Stavros Kois Date: Tue, 3 Dec 2024 14:57:52 +0200 Subject: [PATCH 07/19] lib --- ix-dev/community/wordpress/app.yaml | 48 +- .../library/base_v2_0_31/__init__.py | 0 .../templates/library/base_v2_0_31/configs.py | 86 +++ .../library/base_v2_0_31/container.py | 317 +++++++++ .../templates/library/base_v2_0_31/depends.py | 34 + .../templates/library/base_v2_0_31/deploy.py | 24 + .../templates/library/base_v2_0_31/deps.py | 454 ++++++++++++ .../templates/library/base_v2_0_31/device.py | 31 + .../templates/library/base_v2_0_31/devices.py | 66 ++ .../templates/library/base_v2_0_31/dns.py | 79 +++ .../library/base_v2_0_31/environment.py | 109 +++ .../templates/library/base_v2_0_31/error.py | 4 + .../library/base_v2_0_31/formatter.py | 26 + .../library/base_v2_0_31/functions.py | 148 ++++ .../library/base_v2_0_31/healthcheck.py | 193 +++++ .../templates/library/base_v2_0_31/labels.py | 37 + .../templates/library/base_v2_0_31/notes.py | 70 ++ .../templates/library/base_v2_0_31/portal.py | 22 + .../templates/library/base_v2_0_31/portals.py | 28 + .../templates/library/base_v2_0_31/ports.py | 68 ++ .../templates/library/base_v2_0_31/render.py | 89 +++ .../library/base_v2_0_31/resources.py | 115 +++ .../templates/library/base_v2_0_31/restart.py | 25 + .../templates/library/base_v2_0_31/storage.py | 116 +++ .../templates/library/base_v2_0_31/sysctls.py | 38 + .../library/base_v2_0_31/tests/__init__.py | 0 .../base_v2_0_31/tests/test_build_image.py | 49 ++ .../base_v2_0_31/tests/test_configs.py | 63 ++ .../base_v2_0_31/tests/test_container.py | 324 +++++++++ .../base_v2_0_31/tests/test_depends.py | 54 ++ .../library/base_v2_0_31/tests/test_deps.py | 380 ++++++++++ .../library/base_v2_0_31/tests/test_device.py | 121 ++++ .../library/base_v2_0_31/tests/test_dns.py | 64 ++ .../base_v2_0_31/tests/test_environment.py | 184 +++++ .../base_v2_0_31/tests/test_formatter.py | 13 + .../base_v2_0_31/tests/test_functions.py | 88 +++ .../base_v2_0_31/tests/test_healthcheck.py | 187 +++++ .../library/base_v2_0_31/tests/test_labels.py | 88 +++ .../library/base_v2_0_31/tests/test_notes.py | 213 ++++++ .../library/base_v2_0_31/tests/test_portal.py | 75 ++ .../library/base_v2_0_31/tests/test_ports.py | 110 +++ .../library/base_v2_0_31/tests/test_render.py | 37 + .../base_v2_0_31/tests/test_resources.py | 140 ++++ .../base_v2_0_31/tests/test_restart.py | 57 ++ .../base_v2_0_31/tests/test_sysctls.py | 62 ++ .../base_v2_0_31/tests/test_volumes.py | 666 ++++++++++++++++++ .../library/base_v2_0_31/validations.py | 227 ++++++ .../library/base_v2_0_31/volume_mount.py | 92 +++ .../base_v2_0_31/volume_mount_types.py | 72 ++ .../library/base_v2_0_31/volume_sources.py | 106 +++ .../library/base_v2_0_31/volume_types.py | 133 ++++ .../templates/library/base_v2_0_31/volumes.py | 66 ++ 52 files changed, 5874 insertions(+), 24 deletions(-) create mode 100644 ix-dev/community/wordpress/templates/library/base_v2_0_31/__init__.py create mode 100644 ix-dev/community/wordpress/templates/library/base_v2_0_31/configs.py create mode 100644 ix-dev/community/wordpress/templates/library/base_v2_0_31/container.py create mode 100644 ix-dev/community/wordpress/templates/library/base_v2_0_31/depends.py create mode 100644 ix-dev/community/wordpress/templates/library/base_v2_0_31/deploy.py create mode 100644 ix-dev/community/wordpress/templates/library/base_v2_0_31/deps.py create mode 100644 ix-dev/community/wordpress/templates/library/base_v2_0_31/device.py create mode 100644 ix-dev/community/wordpress/templates/library/base_v2_0_31/devices.py create mode 100644 ix-dev/community/wordpress/templates/library/base_v2_0_31/dns.py create mode 100644 ix-dev/community/wordpress/templates/library/base_v2_0_31/environment.py create mode 100644 ix-dev/community/wordpress/templates/library/base_v2_0_31/error.py create mode 100644 ix-dev/community/wordpress/templates/library/base_v2_0_31/formatter.py create mode 100644 ix-dev/community/wordpress/templates/library/base_v2_0_31/functions.py create mode 100644 ix-dev/community/wordpress/templates/library/base_v2_0_31/healthcheck.py create mode 100644 ix-dev/community/wordpress/templates/library/base_v2_0_31/labels.py create mode 100644 ix-dev/community/wordpress/templates/library/base_v2_0_31/notes.py create mode 100644 ix-dev/community/wordpress/templates/library/base_v2_0_31/portal.py create mode 100644 ix-dev/community/wordpress/templates/library/base_v2_0_31/portals.py create mode 100644 ix-dev/community/wordpress/templates/library/base_v2_0_31/ports.py create mode 100644 ix-dev/community/wordpress/templates/library/base_v2_0_31/render.py create mode 100644 ix-dev/community/wordpress/templates/library/base_v2_0_31/resources.py create mode 100644 ix-dev/community/wordpress/templates/library/base_v2_0_31/restart.py create mode 100644 ix-dev/community/wordpress/templates/library/base_v2_0_31/storage.py create mode 100644 ix-dev/community/wordpress/templates/library/base_v2_0_31/sysctls.py create mode 100644 ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/__init__.py create mode 100644 ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_build_image.py create mode 100644 ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_configs.py create mode 100644 ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_container.py create mode 100644 ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_depends.py create mode 100644 ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_deps.py create mode 100644 ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_device.py create mode 100644 ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_dns.py create mode 100644 ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_environment.py create mode 100644 ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_formatter.py create mode 100644 ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_functions.py create mode 100644 ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_healthcheck.py create mode 100644 ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_labels.py create mode 100644 ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_notes.py create mode 100644 ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_portal.py create mode 100644 ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_ports.py create mode 100644 ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_render.py create mode 100644 ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_resources.py create mode 100644 ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_restart.py create mode 100644 ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_sysctls.py create mode 100644 ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_volumes.py create mode 100644 ix-dev/community/wordpress/templates/library/base_v2_0_31/validations.py create mode 100644 ix-dev/community/wordpress/templates/library/base_v2_0_31/volume_mount.py create mode 100644 ix-dev/community/wordpress/templates/library/base_v2_0_31/volume_mount_types.py create mode 100644 ix-dev/community/wordpress/templates/library/base_v2_0_31/volume_sources.py create mode 100644 ix-dev/community/wordpress/templates/library/base_v2_0_31/volume_types.py create mode 100644 ix-dev/community/wordpress/templates/library/base_v2_0_31/volumes.py diff --git a/ix-dev/community/wordpress/app.yaml b/ix-dev/community/wordpress/app.yaml index 27205dfb88..5b110e80f8 100644 --- a/ix-dev/community/wordpress/app.yaml +++ b/ix-dev/community/wordpress/app.yaml @@ -1,41 +1,41 @@ app_version: 6.7.1 capabilities: - - description: Wordpress requires this ability to bind to port 80 within the container. - name: NET_BIND_SERVICE +- description: Wordpress requires this ability to bind to port 80 within the container. + name: NET_BIND_SERVICE categories: - - productivity +- productivity description: Wordpress is a web content management system home: https://wordpress.org host_mounts: [] icon: https://media.sys.truenas.net/apps/wordpress/icons/icon.png keywords: - - cms - - blog +- cms +- blog lib_version: 2.0.31 -lib_version_hash: a87750c752394f7bf0eb461678564a9ef99bc4e4541787ef25e7be1095a6f879 +lib_version_hash: e61b4db536830d207e591fea73037ca3f335da01a7e4073bb37392d1ede15873 maintainers: - - email: dev@ixsystems.com - name: truenas - url: https://www.truenas.com/ +- email: dev@ixsystems.com + name: truenas + url: https://www.truenas.com/ name: wordpress run_as_context: - - description: Wordpress runs as a non-root user. - gid: 33 - group_name: www-data - uid: 33 - user_name: www-data - - description: MariaDB runs as non-root user. - gid: 999 - group_name: mariadb - uid: 999 - user_name: mariadb +- description: Wordpress runs as a non-root user. + gid: 33 + group_name: www-data + uid: 33 + user_name: www-data +- description: MariaDB runs as non-root user. + gid: 999 + group_name: mariadb + uid: 999 + user_name: mariadb screenshots: - - https://media.sys.truenas.net/apps/wordpress/screenshots/screenshot1.png - - https://media.sys.truenas.net/apps/wordpress/screenshots/screenshot2.png - - https://media.sys.truenas.net/apps/wordpress/screenshots/screenshot3.png - - https://media.sys.truenas.net/apps/wordpress/screenshots/screenshot4.png +- https://media.sys.truenas.net/apps/wordpress/screenshots/screenshot1.png +- https://media.sys.truenas.net/apps/wordpress/screenshots/screenshot2.png +- https://media.sys.truenas.net/apps/wordpress/screenshots/screenshot3.png +- https://media.sys.truenas.net/apps/wordpress/screenshots/screenshot4.png sources: - - https://hub.docker.com/_/wordpress +- https://hub.docker.com/_/wordpress title: Wordpress train: community version: 1.1.0 diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/__init__.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/configs.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/configs.py new file mode 100644 index 0000000000..b76f4b169c --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_0_31/configs.py @@ -0,0 +1,86 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class Configs: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._configs: dict[str, dict] = {} + + def add(self, name: str, data: str): + if not isinstance(data, str): + raise RenderError(f"Expected [data] to be a string, got [{type(data)}]") + + if name not in self._configs: + self._configs[name] = {"name": name, "data": data} + return + + if data == self._configs[name]["data"]: + return + + raise RenderError(f"Config [{name}] already added with different data") + + def has_configs(self): + return bool(self._configs) + + def render(self): + return { + c["name"]: {"content": escape_dollar(c["data"])} + for c in sorted(self._configs.values(), key=lambda c: c["name"]) + } + + +class ContainerConfigs: + def __init__(self, render_instance: "Render", configs: Configs): + self._render_instance = render_instance + self.top_level_configs: Configs = configs + self.container_configs: set[ContainerConfig] = set() + + def add(self, name: str, data: str, target: str, mode: str = ""): + self.top_level_configs.add(name, data) + + if target == "": + raise RenderError(f"Expected [target] to be set for config [{name}]") + if mode != "": + mode = valid_octal_mode_or_raise(mode) + + if target in [c.target for c in self.container_configs]: + raise RenderError(f"Target [{target}] already used for another config") + target = valid_fs_path_or_raise(target) + self.container_configs.add(ContainerConfig(self._render_instance, name, target, mode)) + + def has_configs(self): + return bool(self.container_configs) + + def render(self): + return [c.render() for c in sorted(self.container_configs, key=lambda c: c.source)] + + +class ContainerConfig: + def __init__(self, render_instance: "Render", source: str, target: str, mode: str): + self._render_instance = render_instance + self.source = source + self.target = target + self.mode = mode + + def render(self): + result: dict[str, str | int] = { + "source": self.source, + "target": self.target, + } + + if self.mode: + result["mode"] = int(self.mode, 8) + + return result diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/container.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/container.py new file mode 100644 index 0000000000..a95e76734c --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_0_31/container.py @@ -0,0 +1,317 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + "cap_drop": sorted(self._cap_drop), + "healthcheck": self.healthcheck.render(), + } + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/depends.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/depends.py new file mode 100644 index 0000000000..4e057cf085 --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_0_31/depends.py @@ -0,0 +1,34 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_depend_condition_or_raise +except ImportError: + from error import RenderError + from validations import valid_depend_condition_or_raise + + +class Depends: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._dependencies: dict[str, str] = {} + + def add_dependency(self, name: str, condition: str): + condition = valid_depend_condition_or_raise(condition) + if name in self._dependencies.keys(): + raise RenderError(f"Dependency [{name}] already added") + if name not in self._render_instance.container_names(): + raise RenderError( + f"Dependency [{name}] not found in defined containers. " + f"Available containers: [{', '.join(self._render_instance.container_names())}]" + ) + self._dependencies[name] = condition + + def has_dependencies(self): + return len(self._dependencies) > 0 + + def render(self): + return {d: {"condition": c} for d, c in self._dependencies.items()} diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/deploy.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/deploy.py new file mode 100644 index 0000000000..894dbc643b --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_0_31/deploy.py @@ -0,0 +1,24 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .resources import Resources +except ImportError: + from resources import Resources + + +class Deploy: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self.resources: Resources = Resources(self._render_instance) + + def has_deploy(self): + return self.resources.has_resources() + + def render(self): + if self.resources.has_resources(): + return {"resources": self.resources.render()} + + return {} diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/deps.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/deps.py new file mode 100644 index 0000000000..b3607fa6ab --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_0_31/deps.py @@ -0,0 +1,454 @@ +import os +import json +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import ( + valid_port_or_raise, + valid_fs_path_or_raise, + valid_octal_mode_or_raise, + valid_redis_password_or_raise, + ) +except ImportError: + from error import RenderError + from validations import ( + valid_port_or_raise, + valid_fs_path_or_raise, + valid_octal_mode_or_raise, + valid_redis_password_or_raise, + ) + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = os.path.join("/mnt/permission", identifier) + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + + c.add_storage("/var/lib/postgresql/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("POSTGRES_USER", config["user"]) + c.environment.add_env("POSTGRES_PASSWORD", config["password"]) + c.environment.add_env("POSTGRES_DB", config["database"]) + c.environment.add_env("POSTGRES_PORT", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def _get_port(self): + return self._config.get("port") or 5432 + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/device.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/device.py new file mode 100644 index 0000000000..bfe97097cb --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_0_31/device.py @@ -0,0 +1,31 @@ +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise, allowed_device_or_raise, valid_cgroup_perm_or_raise +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise, allowed_device_or_raise, valid_cgroup_perm_or_raise + + +class Device: + def __init__(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + hd = valid_fs_path_or_raise(host_device.rstrip("/")) + cd = valid_fs_path_or_raise(container_device.rstrip("/")) + if not hd or not cd: + raise RenderError( + "Expected [host_device] and [container_device] to be set. " + f"Got host_device [{host_device}] and container_device [{container_device}]" + ) + + cgroup_perm = valid_cgroup_perm_or_raise(cgroup_perm) + if not allow_disallowed: + hd = allowed_device_or_raise(hd) + + self.cgroup_perm: str = cgroup_perm + self.host_device: str = hd + self.container_device: str = cd + + def render(self): + result = f"{self.host_device}:{self.container_device}" + if self.cgroup_perm: + result += f":{self.cgroup_perm}" + return result diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/devices.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/devices.py new file mode 100644 index 0000000000..ae22c79d2e --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_0_31/devices.py @@ -0,0 +1,66 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/dns.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/dns.py new file mode 100644 index 0000000000..d3ae7b19fa --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_0_31/dns.py @@ -0,0 +1,79 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import allowed_dns_opt_or_raise +except ImportError: + from error import RenderError + from validations import allowed_dns_opt_or_raise + + +class Dns: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._dns_options: set[str] = set() + self._dns_searches: set[str] = set() + self._dns_nameservers: set[str] = set() + + self._auto_add_dns_opts_from_values() + self._auto_add_dns_searches_from_values() + self._auto_add_dns_nameservers_from_values() + + def _get_dns_opt_keys(self): + return [self._get_key_from_opt(opt) for opt in self._dns_options] + + def _get_key_from_opt(self, opt): + return opt.split(":")[0] + + def _auto_add_dns_opts_from_values(self): + values = self._render_instance.values + for dns_opt in values.get("network", {}).get("dns_opts", []): + self.add_dns_opt(dns_opt) + + def _auto_add_dns_searches_from_values(self): + values = self._render_instance.values + for dns_search in values.get("network", {}).get("dns_searches", []): + self.add_dns_search(dns_search) + + def _auto_add_dns_nameservers_from_values(self): + values = self._render_instance.values + for dns_nameserver in values.get("network", {}).get("dns_nameservers", []): + self.add_dns_nameserver(dns_nameserver) + + def add_dns_search(self, dns_search): + if dns_search in self._dns_searches: + raise RenderError(f"DNS Search [{dns_search}] already added") + self._dns_searches.add(dns_search) + + def add_dns_nameserver(self, dns_nameserver): + if dns_nameserver in self._dns_nameservers: + raise RenderError(f"DNS Nameserver [{dns_nameserver}] already added") + self._dns_nameservers.add(dns_nameserver) + + def add_dns_opt(self, dns_opt): + # eg attempts:3 + key = allowed_dns_opt_or_raise(self._get_key_from_opt(dns_opt)) + if key in self._get_dns_opt_keys(): + raise RenderError(f"DNS Option [{key}] already added") + self._dns_options.add(dns_opt) + + def has_dns_opts(self): + return len(self._dns_options) > 0 + + def has_dns_searches(self): + return len(self._dns_searches) > 0 + + def has_dns_nameservers(self): + return len(self._dns_nameservers) > 0 + + def render_dns_searches(self): + return sorted(self._dns_searches) + + def render_dns_opts(self): + return sorted(self._dns_options) + + def render_dns_nameservers(self): + return sorted(self._dns_nameservers) diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/environment.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/environment.py new file mode 100644 index 0000000000..850a3afd8e --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_0_31/environment.py @@ -0,0 +1,109 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/error.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/error.py new file mode 100644 index 0000000000..aef48d3b02 --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_0_31/error.py @@ -0,0 +1,4 @@ +class RenderError(Exception): + """Base class for exceptions in this module.""" + + pass diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/formatter.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/formatter.py new file mode 100644 index 0000000000..24e882f47a --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_0_31/formatter.py @@ -0,0 +1,26 @@ +import json +import hashlib + + +def escape_dollar(text: str) -> str: + return text.replace("$", "$$") + + +def get_hashed_name_for_volume(prefix: str, config: dict): + config_hash = hashlib.sha256(json.dumps(config).encode("utf-8")).hexdigest() + return f"{prefix}_{config_hash}" + + +def get_hash_with_prefix(prefix: str, data: str): + return f"{prefix}_{hashlib.sha256(data.encode('utf-8')).hexdigest()}" + + +def merge_dicts_no_overwrite(dict1, dict2): + overlapping_keys = dict1.keys() & dict2.keys() + if overlapping_keys: + raise ValueError(f"Merging of dicts failed. Overlapping keys: {overlapping_keys}") + return {**dict1, **dict2} + + +def get_image_with_hashed_data(image: str, data: str): + return get_hash_with_prefix(f"ix-{image}", data) diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/functions.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/functions.py new file mode 100644 index 0000000000..47a2c1233a --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_0_31/functions.py @@ -0,0 +1,148 @@ +import re +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return dict.copy() + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/healthcheck.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/healthcheck.py new file mode 100644 index 0000000000..36ae5d90aa --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_0_31/healthcheck.py @@ -0,0 +1,193 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def render(self): + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/labels.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/labels.py new file mode 100644 index 0000000000..f1e667ba00 --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_0_31/labels.py @@ -0,0 +1,37 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar +except ImportError: + from error import RenderError + from formatter import escape_dollar + + +class Labels: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._labels: dict[str, str] = {} + + def add_label(self, key: str, value: str): + if not key: + raise RenderError("Labels must have a key") + + if key.startswith("com.docker.compose"): + raise RenderError(f"Label [{key}] cannot start with [com.docker.compose] as it is reserved") + + if key in self._labels.keys(): + raise RenderError(f"Label [{key}] already added") + + self._labels[key] = escape_dollar(str(value)) + + def has_labels(self) -> bool: + return bool(self._labels) + + def render(self) -> dict[str, str]: + if not self.has_labels(): + return {} + return {label: value for label, value in sorted(self._labels.items())} diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/notes.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/notes.py new file mode 100644 index 0000000000..4adc50c3d8 --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_0_31/notes.py @@ -0,0 +1,70 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + + +class Notes: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._app_name: str = "" + self._warnings: list[str] = [] + self._deprecations: list[str] = [] + self._header: str = "" + self._body: str = "" + self._footer: str = "" + + self._auto_set_app_name() + self._auto_set_header() + self._auto_set_footer() + + def _auto_set_app_name(self): + app_name = self._render_instance.values.get("ix_context", {}).get("app_metadata", {}).get("name", "") + self._app_name = app_name or "" + + def _auto_set_header(self): + head = "# Welcome to TrueNAS SCALE\n\n" + head += f"Thank you for installing {self._app_name}!\n\n" + self._header = head + + def _auto_set_footer(self): + footer = "## Documentation\n\n" + footer += f"Documentation for {self._app_name} can be found at https://www.truenas.com/docs.\n\n" + footer += "## Bug reports\n\n" + footer += "If you find a bug in this app, please file an issue at\n" + footer += "https://ixsystems.atlassian.net or https://github.com/truenas/apps\n\n" + footer += "## Feature requests or improvements\n\n" + footer += "If you find a feature request for this app, please file an issue at\n" + footer += "https://ixsystems.atlassian.net or https://github.com/truenas/apps\n" + self._footer = footer + + def add_warning(self, warning: str): + self._warnings.append(warning) + + def add_deprecation(self, deprecation: str): + self._deprecations.append(deprecation) + + def set_body(self, body: str): + self._body = body + + def render(self): + result = self._header + + if self._warnings: + result += "## Warnings\n\n" + for warning in self._warnings: + result += f"- {warning}\n" + result += "\n" + + if self._deprecations: + result += "## Deprecations\n\n" + for deprecation in self._deprecations: + result += f"- {deprecation}\n" + result += "\n" + + if self._body: + result += self._body.strip() + "\n\n" + + result += self._footer + + return result diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/portal.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/portal.py new file mode 100644 index 0000000000..cf47163439 --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_0_31/portal.py @@ -0,0 +1,22 @@ +try: + from .validations import valid_portal_scheme_or_raise, valid_http_path_or_raise, valid_port_or_raise +except ImportError: + from validations import valid_portal_scheme_or_raise, valid_http_path_or_raise, valid_port_or_raise + + +class Portal: + def __init__(self, name: str, config: dict): + self._name = name + self._scheme = valid_portal_scheme_or_raise(config.get("scheme", "http")) + self._host = config.get("host", "0.0.0.0") or "0.0.0.0" + self._port = valid_port_or_raise(config.get("port", 0)) + self._path = valid_http_path_or_raise(config.get("path", "/")) + + def render(self): + return { + "name": self._name, + "scheme": self._scheme, + "host": self._host, + "port": self._port, + "path": self._path, + } diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/portals.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/portals.py new file mode 100644 index 0000000000..e106d231e6 --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_0_31/portals.py @@ -0,0 +1,28 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .portal import Portal +except ImportError: + from error import RenderError + from portal import Portal + + +class Portals: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._portals: set[Portal] = set() + + def add_portal(self, config: dict): + name = config.get("name", "Web UI") + + if name in [p._name for p in self._portals]: + raise RenderError(f"Portal [{name}] already added") + + self._portals.add(Portal(name, config)) + + def render(self): + return [p.render() for _, p in sorted([(p._name, p) for p in self._portals])] diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/ports.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/ports.py new file mode 100644 index 0000000000..f11e1481b4 --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_0_31/ports.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import ( + valid_port_or_raise, + valid_port_protocol_or_raise, + valid_port_mode_or_raise, + valid_ip_or_raise, + ) +except ImportError: + from error import RenderError + from validations import ( + valid_port_or_raise, + valid_port_protocol_or_raise, + valid_port_mode_or_raise, + valid_ip_or_raise, + ) + + +class Ports: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._ports: dict[str, dict] = {} + + def add_port(self, host_port: int, container_port: int, config: dict | None = None): + config = config or {} + host_port = valid_port_or_raise(host_port) + container_port = valid_port_or_raise(container_port) + proto = valid_port_protocol_or_raise(config.get("protocol", "tcp")) + mode = valid_port_mode_or_raise(config.get("mode", "ingress")) + host_ip = valid_ip_or_raise(config.get("host_ip", "0.0.0.0")) + + key = f"{host_port}_{host_ip}_{proto}" + if key in self._ports.keys(): + raise RenderError(f"Port [{host_port}/{proto}] already added for [{host_ip}]") + + if host_ip != "0.0.0.0": + # If the port we are adding is not going to use 0.0.0.0 + # Make sure that we don't have already added that port/proto to 0.0.0.0 + search_key = f"{host_port}_0.0.0.0_{proto}" + if search_key in self._ports.keys(): + raise RenderError(f"Cannot bind port [{host_port}/{proto}] to [{host_ip}], already bound to [0.0.0.0]") + elif host_ip == "0.0.0.0": + # If the port we are adding is going to use 0.0.0.0 + # Make sure that we don't have already added that port/proto to a specific ip + for p in self._ports.values(): + if p["published"] == host_port and p["protocol"] == proto: + raise RenderError( + f"Cannot bind port [{host_port}/{proto}] to [{host_ip}], already bound to [{p['host_ip']}]" + ) + + self._ports[key] = { + "published": host_port, + "target": container_port, + "protocol": proto, + "mode": mode, + "host_ip": host_ip, + } + + def has_ports(self): + return len(self._ports) > 0 + + def render(self): + return [config for _, config in sorted(self._ports.items())] diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/render.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/render.py new file mode 100644 index 0000000000..9d8fcc28d5 --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_0_31/render.py @@ -0,0 +1,89 @@ +import copy + +try: + from .configs import Configs + from .container import Container + from .deps import Deps + from .error import RenderError + from .functions import Functions + from .notes import Notes + from .portals import Portals + from .volumes import Volumes +except ImportError: + from configs import Configs + from container import Container + from deps import Deps + from error import RenderError + from functions import Functions + from notes import Notes + from portals import Portals + from volumes import Volumes + + +class Render(object): + def __init__(self, values): + self._containers: dict[str, Container] = {} + self.values = values + self._add_images_internal_use() + # Make a copy after we inject the images + self._original_values: dict = copy.deepcopy(self.values) + + self.deps: Deps = Deps(self) + + self.configs = Configs(render_instance=self) + self.funcs = Functions(render_instance=self).func_map() + self.portals: Portals = Portals(render_instance=self) + self.notes: Notes = Notes(render_instance=self) + self.volumes = Volumes(render_instance=self) + + def _add_images_internal_use(self): + if not self.values.get("images"): + self.values["images"] = {} + + if "python_permissions_image" not in self.values["images"]: + self.values["images"]["python_permissions_image"] = {"repository": "python", "tag": "3.13.0-slim-bookworm"} + + def container_names(self): + return list(self._containers.keys()) + + def add_container(self, name: str, image: str): + name = name.strip() + if not name: + raise RenderError("Container name cannot be empty") + container = Container(self, name, image) + if name in self._containers: + raise RenderError(f"Container {name} already exists.") + self._containers[name] = container + return container + + def render(self): + if self.values != self._original_values: + raise RenderError("Values have been modified since the renderer was created.") + + if not self._containers: + raise RenderError("No containers added.") + + result: dict = { + "x-notes": self.notes.render(), + "x-portals": self.portals.render(), + "services": {c._name: c.render() for c in self._containers.values()}, + } + + # Make sure that after services are rendered + # there are no labels that target a non-existent container + # This is to prevent typos + for label in self.values.get("labels", []): + for c in label.get("containers", []): + if c not in self.container_names(): + raise RenderError(f"Label [{label['key']}] references container [{c}] which does not exist") + + if self.volumes.has_volumes(): + result["volumes"] = self.volumes.render() + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + # if self.networks: + # result["networks"] = {...} + + return result diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/resources.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/resources.py new file mode 100644 index 0000000000..733f43bb6f --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_0_31/resources.py @@ -0,0 +1,115 @@ +import re +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +DEFAULT_CPUS = 2.0 +DEFAULT_MEMORY = 4096 + + +class Resources: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._limits: dict = {} + self._reservations: dict = {} + self._nvidia_ids: set[str] = set() + self._auto_add_cpu_from_values() + self._auto_add_memory_from_values() + self._auto_add_gpus_from_values() + + def _set_cpu(self, cpus: Any): + c = str(cpus) + if not re.match(r"^[1-9][0-9]*(\.[0-9]+)?$", c): + raise RenderError(f"Expected cpus to be a number or a float (minimum 1.0), got [{cpus}]") + self._limits.update({"cpus": c}) + + def _set_memory(self, memory: Any): + m = str(memory) + if not re.match(r"^[1-9][0-9]*$", m): + raise RenderError(f"Expected memory to be a number, got [{memory}]") + self._limits.update({"memory": f"{m}M"}) + + def _auto_add_cpu_from_values(self): + resources = self._render_instance.values.get("resources", {}) + self._set_cpu(resources.get("limits", {}).get("cpus", DEFAULT_CPUS)) + + def _auto_add_memory_from_values(self): + resources = self._render_instance.values.get("resources", {}) + self._set_memory(resources.get("limits", {}).get("memory", DEFAULT_MEMORY)) + + def _auto_add_gpus_from_values(self): + resources = self._render_instance.values.get("resources", {}) + gpus = resources.get("gpus", {}).get("nvidia_gpu_selection", {}) + if not gpus: + return + + for pci, gpu in gpus.items(): + if gpu.get("use_gpu", False): + if not gpu.get("uuid"): + raise RenderError(f"Expected [uuid] to be set for GPU in slot [{pci}] in [nvidia_gpu_selection]") + self._nvidia_ids.add(gpu["uuid"]) + + if self._nvidia_ids: + if not self._reservations: + self._reservations["devices"] = [] + self._reservations["devices"].append( + { + "capabilities": ["gpu"], + "driver": "nvidia", + "device_ids": sorted(self._nvidia_ids), + } + ) + + # This is only used on ix-app that we allow + # disabling cpus and memory. GPUs are only added + # if the user has requested them. + def remove_cpus_and_memory(self): + self._limits.pop("cpus", None) + self._limits.pop("memory", None) + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._reservations.pop("devices", None) + + def set_profile(self, profile: str): + cpu, memory = profile_mapping(profile) + self._set_cpu(cpu) + self._set_memory(memory) + + def has_resources(self): + return len(self._limits) > 0 or len(self._reservations) > 0 + + def has_gpus(self): + gpu_devices = [d for d in self._reservations.get("devices", []) if "gpu" in d["capabilities"]] + return len(gpu_devices) > 0 + + def render(self): + result = {} + if self._limits: + result["limits"] = self._limits + if self._reservations: + result["reservations"] = self._reservations + + return result + + +def profile_mapping(profile: str): + profiles = { + "low": (1, 512), + "medium": (2, 1024), + } + + if profile not in profiles: + raise RenderError( + f"Resource profile [{profile}] is not valid. Valid options are: [{', '.join(profiles.keys())}]" + ) + + return profiles[profile] diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/restart.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/restart.py new file mode 100644 index 0000000000..2f6281af48 --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_0_31/restart.py @@ -0,0 +1,25 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .validations import valid_restart_policy_or_raise +except ImportError: + from validations import valid_restart_policy_or_raise + + +class RestartPolicy: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._policy: str = "unless-stopped" + self._maximum_retry_count: int = 0 + + def set_policy(self, policy: str, maximum_retry_count: int = 0): + self._policy = valid_restart_policy_or_raise(policy, maximum_retry_count) + self._maximum_retry_count = maximum_retry_count + + def render(self): + if self._policy == "on-failure" and self._maximum_retry_count > 0: + return f"{self._policy}:{self._maximum_retry_count}" + return self._policy diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/storage.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_0_31/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/sysctls.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_0_31/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/__init__.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_build_image.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_build_image.py new file mode 100644 index 0000000000..f30c1210ed --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_build_image.py @@ -0,0 +1,49 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_build_image_with_from(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.build_image(["FROM test_image"]) + + +def test_build_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.build_image( + [ + "RUN echo hello", + None, + "", + "RUN echo world", + ] + ) + output = render.render() + assert ( + output["services"]["test_container"]["image"] + == "ix-nginx:latest_4a127145ea4c25511707e57005dd0ed457fe2f4932082c8f9faa339a450b6a99" + ) + assert output["services"]["test_container"]["build"] == { + "tags": ["ix-nginx:latest_4a127145ea4c25511707e57005dd0ed457fe2f4932082c8f9faa339a450b6a99"], + "dockerfile_inline": """FROM nginx:latest +RUN echo hello +RUN echo world +""", + } diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_configs.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_configs.py new file mode 100644 index 0000000000..9049e473ea --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_configs.py @@ -0,0 +1,63 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_duplicate_config_with_different_data(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.configs.add("test_config", "test_data", "/some/path") + with pytest.raises(Exception): + c1.configs.add("test_config", "test_data2", "/some/path") + + +def test_add_config_with_empty_target(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.configs.add("test_config", "test_data", "") + + +def test_add_duplicate_target(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.configs.add("test_config", "test_data", "/some/path") + with pytest.raises(Exception): + c1.configs.add("test_config2", "test_data2", "/some/path") + + +def test_add_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.configs.add("test_config", "$test_data", "/some/path") + output = render.render() + assert output["configs"]["test_config"]["content"] == "$$test_data" + assert output["services"]["test_container"]["configs"] == [{"source": "test_config", "target": "/some/path"}] + + +def test_add_config_with_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.configs.add("test_config", "test_data", "/some/path", "0777") + output = render.render() + assert output["configs"]["test_config"]["content"] == "test_data" + assert output["services"]["test_container"]["configs"] == [ + {"source": "test_config", "target": "/some/path", "mode": 511} + ] diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_container.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_container.py new file mode 100644 index 0000000000..61a22a5df2 --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_container.py @@ -0,0 +1,324 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_depends.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_depends.py new file mode 100644 index 0000000000..a1d8373927 --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_depends.py @@ -0,0 +1,54 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_dependency(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c2 = render.add_container("test_container2", "test_image") + c1.healthcheck.disable() + c2.healthcheck.disable() + c1.depends.add_dependency("test_container2", "service_started") + output = render.render() + assert output["services"]["test_container"]["depends_on"]["test_container2"] == {"condition": "service_started"} + + +def test_add_dependency_invalid_condition(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + render.add_container("test_container2", "test_image") + with pytest.raises(Exception): + c1.depends.add_dependency("test_container2", "invalid_condition") + + +def test_add_dependency_missing_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.depends.add_dependency("test_container2", "service_started") + + +def test_add_dependency_duplicate(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + render.add_container("test_container2", "test_image") + c1.depends.add_dependency("test_container2", "service_started") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.depends.add_dependency("test_container2", "service_started") diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_deps.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_deps.py new file mode 100644 index 0000000000..f9562ba4f2 --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_deps.py @@ -0,0 +1,380 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:latest" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_device.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_device.py new file mode 100644 index 0000000000..7455c829f6 --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_device.py @@ -0,0 +1,121 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_dns.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_dns.py new file mode 100644 index 0000000000..fe6b21e34f --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_dns.py @@ -0,0 +1,64 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_dns_opts(mock_values): + mock_values["network"] = {"dns_opts": ["attempts:3", "opt1", "opt2"]} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["dns_opt"] == ["attempts:3", "opt1", "opt2"] + + +def test_auto_add_dns_searches(mock_values): + mock_values["network"] = {"dns_searches": ["search1", "search2"]} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["dns_search"] == ["search1", "search2"] + + +def test_auto_add_dns_nameservers(mock_values): + mock_values["network"] = {"dns_nameservers": ["nameserver1", "nameserver2"]} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["dns"] == ["nameserver1", "nameserver2"] + + +def test_add_duplicate_dns_nameservers(mock_values): + mock_values["network"] = {"dns_nameservers": ["nameserver1", "nameserver1"]} + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_add_duplicate_dns_searches(mock_values): + mock_values["network"] = {"dns_searches": ["search1", "search1"]} + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_add_duplicate_dns_opts(mock_values): + mock_values["network"] = {"dns_opts": ["attempts:3", "attempts:5"]} + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_environment.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_environment.py new file mode 100644 index 0000000000..209f67551b --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_environment.py @@ -0,0 +1,184 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_formatter.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_formatter.py new file mode 100644 index 0000000000..843cf65d2e --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_formatter.py @@ -0,0 +1,13 @@ +from formatter import escape_dollar + + +def test_escape_dollar(): + cases = [ + {"input": "test", "expected": "test"}, + {"input": "$test", "expected": "$$test"}, + {"input": "$$test", "expected": "$$$$test"}, + {"input": "$$$test", "expected": "$$$$$$test"}, + {"input": "$test$", "expected": "$$test$$"}, + ] + for case in cases: + assert escape_dollar(case["input"]) == case["expected"] diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_functions.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_healthcheck.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_healthcheck.py new file mode 100644 index 0000000000..fbd488ece4 --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_healthcheck.py @@ -0,0 +1,187 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_labels.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_labels.py new file mode 100644 index 0000000000..ffa21eceac --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_labels.py @@ -0,0 +1,88 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_disallowed_label(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.labels.add_label("com.docker.compose.service", "test_service") + + +def test_add_duplicate_label(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.labels.add_label("my.custom.label", "test_value") + with pytest.raises(Exception): + c1.labels.add_label("my.custom.label", "test_value1") + + +def test_add_label_on_non_existing_container(mock_values): + mock_values["labels"] = [ + { + "key": "my.custom.label1", + "value": "test_value1", + "containers": ["test_container", "test_container2"], + }, + ] + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.render() + + +def test_add_label(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.labels.add_label("my.custom.label1", "test_value1") + c1.labels.add_label("my.custom.label2", "test_value2") + output = render.render() + assert output["services"]["test_container"]["labels"] == { + "my.custom.label1": "test_value1", + "my.custom.label2": "test_value2", + } + + +def test_auto_add_labels(mock_values): + mock_values["labels"] = [ + { + "key": "my.custom.label1", + "value": "test_value1", + "containers": ["test_container", "test_container2"], + }, + { + "key": "my.custom.label2", + "value": "test_value2", + "containers": ["test_container"], + }, + ] + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c2 = render.add_container("test_container2", "test_image") + c1.healthcheck.disable() + c2.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["labels"] == { + "my.custom.label1": "test_value1", + "my.custom.label2": "test_value2", + } + assert output["services"]["test_container2"]["labels"] == { + "my.custom.label1": "test_value1", + } diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_notes.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_notes.py new file mode 100644 index 0000000000..3613445385 --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_notes.py @@ -0,0 +1,213 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "ix_context": { + "app_metadata": { + "name": "test_app", + } + }, + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_notes(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert ( + output["x-notes"] + == """# Welcome to TrueNAS SCALE + +Thank you for installing test_app! + +## Documentation + +Documentation for test_app can be found at https://www.truenas.com/docs. + +## Bug reports + +If you find a bug in this app, please file an issue at +https://ixsystems.atlassian.net or https://github.com/truenas/apps + +## Feature requests or improvements + +If you find a feature request for this app, please file an issue at +https://ixsystems.atlassian.net or https://github.com/truenas/apps +""" + ) + + +def test_notes_with_warnings(mock_values): + render = Render(mock_values) + render.notes.add_warning("this is not properly configured. fix it now!") + render.notes.add_warning("that is not properly configured. fix it later!") + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert ( + output["x-notes"] + == """# Welcome to TrueNAS SCALE + +Thank you for installing test_app! + +## Warnings + +- this is not properly configured. fix it now! +- that is not properly configured. fix it later! + +## Documentation + +Documentation for test_app can be found at https://www.truenas.com/docs. + +## Bug reports + +If you find a bug in this app, please file an issue at +https://ixsystems.atlassian.net or https://github.com/truenas/apps + +## Feature requests or improvements + +If you find a feature request for this app, please file an issue at +https://ixsystems.atlassian.net or https://github.com/truenas/apps +""" + ) + + +def test_notes_with_deprecations(mock_values): + render = Render(mock_values) + render.notes.add_deprecation("this is will be removed later. fix it now!") + render.notes.add_deprecation("that is will be removed later. fix it later!") + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert ( + output["x-notes"] + == """# Welcome to TrueNAS SCALE + +Thank you for installing test_app! + +## Deprecations + +- this is will be removed later. fix it now! +- that is will be removed later. fix it later! + +## Documentation + +Documentation for test_app can be found at https://www.truenas.com/docs. + +## Bug reports + +If you find a bug in this app, please file an issue at +https://ixsystems.atlassian.net or https://github.com/truenas/apps + +## Feature requests or improvements + +If you find a feature request for this app, please file an issue at +https://ixsystems.atlassian.net or https://github.com/truenas/apps +""" + ) + + +def test_notes_with_body(mock_values): + render = Render(mock_values) + render.notes.set_body( + """## Additional info + +Some info +some other info. +""" + ) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert ( + output["x-notes"] + == """# Welcome to TrueNAS SCALE + +Thank you for installing test_app! + +## Additional info + +Some info +some other info. + +## Documentation + +Documentation for test_app can be found at https://www.truenas.com/docs. + +## Bug reports + +If you find a bug in this app, please file an issue at +https://ixsystems.atlassian.net or https://github.com/truenas/apps + +## Feature requests or improvements + +If you find a feature request for this app, please file an issue at +https://ixsystems.atlassian.net or https://github.com/truenas/apps +""" + ) + + +def test_notes_all(mock_values): + render = Render(mock_values) + render.notes.add_warning("this is not properly configured. fix it now!") + render.notes.add_warning("that is not properly configured. fix it later!") + render.notes.add_deprecation("this is will be removed later. fix it now!") + render.notes.add_deprecation("that is will be removed later. fix it later!") + render.notes.set_body( + """## Additional info + +Some info +some other info. +""" + ) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert ( + output["x-notes"] + == """# Welcome to TrueNAS SCALE + +Thank you for installing test_app! + +## Warnings + +- this is not properly configured. fix it now! +- that is not properly configured. fix it later! + +## Deprecations + +- this is will be removed later. fix it now! +- that is will be removed later. fix it later! + +## Additional info + +Some info +some other info. + +## Documentation + +Documentation for test_app can be found at https://www.truenas.com/docs. + +## Bug reports + +If you find a bug in this app, please file an issue at +https://ixsystems.atlassian.net or https://github.com/truenas/apps + +## Feature requests or improvements + +If you find a feature request for this app, please file an issue at +https://ixsystems.atlassian.net or https://github.com/truenas/apps +""" + ) diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_portal.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_portal.py new file mode 100644 index 0000000000..aebd9425c9 --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_portal.py @@ -0,0 +1,75 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_no_portals(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["x-portals"] == [] + + +def test_add_portal(mock_values): + render = Render(mock_values) + render.portals.add_portal({"scheme": "http", "path": "/", "port": 8080}) + render.portals.add_portal({"name": "Other Portal", "scheme": "https", "path": "/", "port": 8443}) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["x-portals"] == [ + {"name": "Other Portal", "scheme": "https", "host": "0.0.0.0", "port": 8443, "path": "/"}, + {"name": "Web UI", "scheme": "http", "host": "0.0.0.0", "port": 8080, "path": "/"}, + ] + + +def test_add_duplicate_portal(mock_values): + render = Render(mock_values) + render.portals.add_portal({"scheme": "http", "path": "/", "port": 8080}) + with pytest.raises(Exception): + render.portals.add_portal({"scheme": "http", "path": "/", "port": 8080}) + + +def test_add_duplicate_portal_with_explicit_name(mock_values): + render = Render(mock_values) + render.portals.add_portal({"name": "Some Portal", "scheme": "http", "path": "/", "port": 8080}) + with pytest.raises(Exception): + render.portals.add_portal({"name": "Some Portal", "scheme": "http", "path": "/", "port": 8080}) + + +def test_add_portal_with_invalid_scheme(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.portals.add_portal({"scheme": "invalid_scheme", "path": "/", "port": 8080}) + + +def test_add_portal_with_invalid_path(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.portals.add_portal({"scheme": "http", "path": "invalid_path", "port": 8080}) + + +def test_add_portal_with_invalid_path_double_slash(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.portals.add_portal({"scheme": "http", "path": "/some//path", "port": 8080}) + + +def test_add_portal_with_invalid_port(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.portals.add_portal({"scheme": "http", "path": "/", "port": -1}) diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_ports.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_ports.py new file mode 100644 index 0000000000..a4c923ca1d --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_ports.py @@ -0,0 +1,110 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_ports(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.ports.add_port(8081, 8080) + c1.ports.add_port(8082, 8080, {"protocol": "udp"}) + output = render.render() + assert output["services"]["test_container"]["ports"] == [ + {"published": 8081, "target": 8080, "protocol": "tcp", "mode": "ingress", "host_ip": "0.0.0.0"}, + {"published": 8082, "target": 8080, "protocol": "udp", "mode": "ingress", "host_ip": "0.0.0.0"}, + ] + + +def test_add_duplicate_ports(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.ports.add_port(8081, 8080) + c1.ports.add_port(8081, 8080, {"protocol": "udp"}) # This should not raise + with pytest.raises(Exception): + c1.ports.add_port(8081, 8080) + + +def test_add_duplicate_ports_with_different_host_ip(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.ports.add_port(8081, 8080, {"host_ip": "192.168.1.10"}) + c1.ports.add_port(8081, 8080, {"host_ip": "192.168.1.11"}) + output = render.render() + assert output["services"]["test_container"]["ports"] == [ + {"published": 8081, "target": 8080, "protocol": "tcp", "mode": "ingress", "host_ip": "192.168.1.10"}, + {"published": 8081, "target": 8080, "protocol": "tcp", "mode": "ingress", "host_ip": "192.168.1.11"}, + ] + + +def test_add_duplicate_ports_to_specific_host_ip_binds_to_0_0_0_0(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.ports.add_port(8081, 8080, {"host_ip": "192.168.1.10"}) + with pytest.raises(Exception): + c1.ports.add_port(8081, 8080, {"host_ip": "0.0.0.0"}) + + +def test_add_duplicate_ports_to_0_0_0_0_binds_to_specific_host_ip(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.ports.add_port(8081, 8080, {"host_ip": "0.0.0.0"}) + with pytest.raises(Exception): + c1.ports.add_port(8081, 8080, {"host_ip": "192.168.1.10"}) + + +def test_add_ports_with_invalid_protocol(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.ports.add_port(8081, 8080, {"protocol": "invalid_protocol"}) + + +def test_add_ports_with_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.ports.add_port(8081, 8080, {"mode": "invalid_mode"}) + + +def test_add_ports_with_invalid_ip(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.ports.add_port(8081, 8080, {"host_ip": "invalid_ip"}) + + +def test_add_ports_with_invalid_host_port(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.ports.add_port(-1, 8080) + + +def test_add_ports_with_invalid_container_port(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.ports.add_port(8081, -1) diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_render.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_render.py new file mode 100644 index 0000000000..60dc00679e --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_render.py @@ -0,0 +1,37 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_values_cannot_be_modified(mock_values): + render = Render(mock_values) + render.values["test"] = "test" + with pytest.raises(Exception): + render.render() + + +def test_duplicate_containers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_no_containers(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_resources.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_resources.py new file mode 100644 index 0000000000..cd83d164e5 --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_resources.py @@ -0,0 +1,140 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_automatically_add_cpu(mock_values): + mock_values["resources"] = {"limits": {"cpus": 1.0}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["deploy"]["resources"]["limits"]["cpus"] == "1.0" + + +def test_invalid_cpu(mock_values): + mock_values["resources"] = {"limits": {"cpus": "invalid"}} + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_automatically_add_memory(mock_values): + mock_values["resources"] = {"limits": {"memory": 1024}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["deploy"]["resources"]["limits"]["memory"] == "1024M" + + +def test_invalid_memory(mock_values): + mock_values["resources"] = {"limits": {"memory": "invalid"}} + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_automatically_add_gpus(mock_values): + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + devices = output["services"]["test_container"]["deploy"]["resources"]["reservations"]["devices"] + assert len(devices) == 1 + assert devices[0] == { + "capabilities": ["gpu"], + "driver": "nvidia", + "device_ids": ["uuid_0", "uuid_1"], + } + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_gpu_without_uuid(mock_values): + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_remove_cpus_and_memory_with_gpus(mock_values): + mock_values["resources"] = {"gpus": {"nvidia_gpu_selection": {"pci_slot_0": {"uuid": "uuid_1", "use_gpu": True}}}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.deploy.resources.remove_cpus_and_memory() + output = render.render() + assert "limits" not in output["services"]["test_container"]["deploy"]["resources"] + devices = output["services"]["test_container"]["deploy"]["resources"]["reservations"]["devices"] + assert len(devices) == 1 + assert devices[0] == { + "capabilities": ["gpu"], + "driver": "nvidia", + "device_ids": ["uuid_1"], + } + + +def test_remove_cpus_and_memory(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.deploy.resources.remove_cpus_and_memory() + output = render.render() + assert "deploy" not in output["services"]["test_container"] + + +def test_remove_devices(mock_values): + mock_values["resources"] = {"gpus": {"nvidia_gpu_selection": {"pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}}}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.deploy.resources.remove_devices() + output = render.render() + assert "reservations" not in output["services"]["test_container"]["deploy"]["resources"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_set_profile(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.deploy.resources.set_profile("low") + output = render.render() + assert output["services"]["test_container"]["deploy"]["resources"]["limits"]["cpus"] == "1" + assert output["services"]["test_container"]["deploy"]["resources"]["limits"]["memory"] == "512M" + + +def test_set_profile_invalid_profile(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.deploy.resources.set_profile("invalid_profile") diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_restart.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_restart.py new file mode 100644 index 0000000000..06b2975590 --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_restart.py @@ -0,0 +1,57 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_invalid_restart_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.restart.set_policy("invalid_policy") + + +def test_valid_restart_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.restart.set_policy("on-failure") + output = render.render() + assert output["services"]["test_container"]["restart"] == "on-failure" + + +def test_valid_restart_policy_with_maximum_retry_count(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.restart.set_policy("on-failure", 10) + output = render.render() + assert output["services"]["test_container"]["restart"] == "on-failure:10" + + +def test_invalid_restart_policy_with_maximum_retry_count(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.restart.set_policy("on-failure", maximum_retry_count=-1) + + +def test_invalid_restart_policy_with_maximum_retry_count_and_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.restart.set_policy("always", maximum_retry_count=10) diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_sysctls.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_volumes.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/validations.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/validations.py new file mode 100644 index 0000000000..13f155dfdb --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_0_31/validations.py @@ -0,0 +1,227 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/volume_mount.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/volume_mount.py new file mode 100644 index 0000000000..aadd077750 --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_0_31/volume_mount.py @@ -0,0 +1,92 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .formatter import merge_dicts_no_overwrite + from .volume_mount_types import BindMountType, VolumeMountType, TmpfsMountType + from .volume_sources import HostPathSource, IxVolumeSource, CifsSource, NfsSource, VolumeSource +except ImportError: + from error import RenderError + from formatter import merge_dicts_no_overwrite + from volume_mount_types import BindMountType, VolumeMountType, TmpfsMountType + from volume_sources import HostPathSource, IxVolumeSource, CifsSource, NfsSource, VolumeSource + + +class VolumeMount: + def __init__(self, render_instance: "Render", mount_path: str, config: "IxStorage"): + self._render_instance = render_instance + self.mount_path: str = mount_path + + storage_type: str = config.get("type", "") + if not storage_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match storage_type: + case "host_path": + spec_type = "bind" + mount_config = config.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + mount_type_specific_definition = BindMountType(self._render_instance, mount_config).render() + source = HostPathSource(self._render_instance, mount_config).get() + case "ix_volume": + spec_type = "bind" + mount_config = config.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + mount_type_specific_definition = BindMountType(self._render_instance, mount_config).render() + source = IxVolumeSource(self._render_instance, mount_config).get() + case "tmpfs": + spec_type = "tmpfs" + mount_config = config.get("tmpfs_config", {}) + mount_type_specific_definition = TmpfsMountType(self._render_instance, mount_config).render() + source = None + case "nfs": + spec_type = "volume" + mount_config = config.get("nfs_config") + if mount_config is None: + raise RenderError("Expected [nfs_config] to be set for [nfs] type.") + mount_type_specific_definition = VolumeMountType(self._render_instance, mount_config).render() + source = NfsSource(self._render_instance, mount_config).get() + case "cifs": + spec_type = "volume" + mount_config = config.get("cifs_config") + if mount_config is None: + raise RenderError("Expected [cifs_config] to be set for [cifs] type.") + mount_type_specific_definition = VolumeMountType(self._render_instance, mount_config).render() + source = CifsSource(self._render_instance, mount_config).get() + case "volume": + spec_type = "volume" + mount_config = config.get("volume_config") + if mount_config is None: + raise RenderError("Expected [volume_config] to be set for [volume] type.") + mount_type_specific_definition = VolumeMountType(self._render_instance, mount_config).render() + source = VolumeSource(self._render_instance, mount_config).get() + case "temporary": + spec_type = "volume" + mount_config = config.get("volume_config") + if mount_config is None: + raise RenderError("Expected [volume_config] to be set for [temporary] type.") + mount_type_specific_definition = VolumeMountType(self._render_instance, mount_config).render() + source = VolumeSource(self._render_instance, mount_config).get() + case "anonymous": + spec_type = "volume" + mount_config = config.get("volume_config") or {} + mount_type_specific_definition = VolumeMountType(self._render_instance, mount_config).render() + source = None + case _: + raise RenderError(f"Storage type [{storage_type}] is not supported for volume mounts.") + + common_spec = {"type": spec_type, "target": self.mount_path, "read_only": config.get("read_only", False)} + if source is not None: + common_spec["source"] = source + self._render_instance.volumes.add_volume(source, storage_type, mount_config) # type: ignore + + self.volume_mount_spec = merge_dicts_no_overwrite(common_spec, mount_type_specific_definition) + + def render(self) -> dict: + return self.volume_mount_spec diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/volume_mount_types.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/volume_mount_types.py new file mode 100644 index 0000000000..00a0ec3a18 --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_0_31/volume_mount_types.py @@ -0,0 +1,72 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorageTmpfsConfig, IxStorageVolumeConfig, IxStorageBindLikeConfigs + + +try: + from .error import RenderError + from .validations import valid_host_path_propagation, valid_octal_mode_or_raise +except ImportError: + from error import RenderError + from validations import valid_host_path_propagation, valid_octal_mode_or_raise + + +class TmpfsMountType: + def __init__(self, render_instance: "Render", config: "IxStorageTmpfsConfig"): + self._render_instance = render_instance + self.spec = {"tmpfs": {}} + size = config.get("size", None) + mode = config.get("mode", None) + + if size is not None: + if not isinstance(size, int): + raise RenderError(f"Expected [size] to be an integer for [tmpfs] type, got [{size}]") + if not size > 0: + raise RenderError(f"Expected [size] to be greater than 0 for [tmpfs] type, got [{size}]") + # Convert Mebibytes to Bytes + self.spec["tmpfs"]["size"] = size * 1024 * 1024 + + if mode is not None: + mode = valid_octal_mode_or_raise(mode) + self.spec["tmpfs"]["mode"] = int(mode, 8) + + if not self.spec["tmpfs"]: + self.spec.pop("tmpfs") + + def render(self) -> dict: + """Render the tmpfs mount specification.""" + return self.spec + + +class BindMountType: + def __init__(self, render_instance: "Render", config: "IxStorageBindLikeConfigs"): + self._render_instance = render_instance + self.spec: dict = {} + + propagation = valid_host_path_propagation(config.get("propagation", "rprivate")) + create_host_path = config.get("create_host_path", False) + + self.spec: dict = { + "bind": { + "create_host_path": create_host_path, + "propagation": propagation, + } + } + + def render(self) -> dict: + """Render the bind mount specification.""" + return self.spec + + +class VolumeMountType: + def __init__(self, render_instance: "Render", config: "IxStorageVolumeConfig"): + self._render_instance = render_instance + self.spec: dict = {} + + self.spec: dict = {"volume": {"nocopy": config.get("nocopy", False)}} + + def render(self) -> dict: + """Render the volume mount specification.""" + return self.spec diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/volume_sources.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/volume_sources.py new file mode 100644 index 0000000000..c33fe55ea1 --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_0_31/volume_sources.py @@ -0,0 +1,106 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorageHostPathConfig, IxStorageIxVolumeConfig, IxStorageVolumeConfig + +try: + from .error import RenderError + from .formatter import get_hashed_name_for_volume + from .validations import valid_fs_path_or_raise +except ImportError: + from error import RenderError + from formatter import get_hashed_name_for_volume + from validations import valid_fs_path_or_raise + + +class HostPathSource: + def __init__(self, render_instance: "Render", config: "IxStorageHostPathConfig"): + self._render_instance = render_instance + self.source: str = "" + + if not config: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + + path = "" + if config.get("acl_enable", False): + acl_path = config.get("acl", {}).get("path") + if not acl_path: + raise RenderError("Expected [host_path_config.acl.path] to be set for [host_path] type.") + path = valid_fs_path_or_raise(acl_path) + else: + path = valid_fs_path_or_raise(config.get("path", "")) + + self.source = path.rstrip("/") + + def get(self): + return self.source + + +class IxVolumeSource: + def __init__(self, render_instance: "Render", config: "IxStorageIxVolumeConfig"): + self._render_instance = render_instance + self.source: str = "" + + if not config: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + dataset_name = config.get("dataset_name") + if not dataset_name: + raise RenderError("Expected [ix_volume_config.dataset_name] to be set for [ix_volume] type.") + + ix_volumes = self._render_instance.values.get("ix_volumes", {}) + if dataset_name not in ix_volumes: + available = ", ".join(ix_volumes.keys()) + raise RenderError( + f"Expected the key [{dataset_name}] to be set in [ix_volumes] for [ix_volume] type. " + f"Available keys: [{available}]." + ) + + self.source = valid_fs_path_or_raise(ix_volumes[dataset_name].rstrip("/")) + + def get(self): + return self.source + + +class CifsSource: + def __init__(self, render_instance: "Render", config: dict): + self._render_instance = render_instance + self.source: str = "" + + if not config: + raise RenderError("Expected [cifs_config] to be set for [cifs] type.") + self.source = get_hashed_name_for_volume("cifs", config) + + def get(self): + return self.source + + +class NfsSource: + def __init__(self, render_instance: "Render", config: dict): + self._render_instance = render_instance + self.source: str = "" + + if not config: + raise RenderError("Expected [nfs_config] to be set for [nfs] type.") + self.source = get_hashed_name_for_volume("nfs", config) + + def get(self): + return self.source + + +class VolumeSource: + def __init__(self, render_instance: "Render", config: "IxStorageVolumeConfig"): + self._render_instance = render_instance + self.source: str = "" + + if not config: + raise RenderError("Expected [volume_config] to be set for [volume] type.") + + volume_name: str = config.get("volume_name", "") + if not volume_name: + raise RenderError("Expected [volume_config.volume_name] to be set for [volume] type.") + + self.source = volume_name + + def get(self): + return self.source diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/volume_types.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/volume_types.py new file mode 100644 index 0000000000..4ccea08f83 --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_0_31/volume_types.py @@ -0,0 +1,133 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorageNfsConfig, IxStorageCifsConfig, IxStorageVolumeConfig + + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_fs_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_fs_path_or_raise + + +class NfsVolume: + def __init__(self, render_instance: "Render", config: "IxStorageNfsConfig"): + self._render_instance = render_instance + + if not config: + raise RenderError("Expected [nfs_config] to be set for [nfs] type") + + required_keys = ["server", "path"] + for key in required_keys: + if not config.get(key): + raise RenderError(f"Expected [{key}] to be set for [nfs] type") + + opts = [f"addr={config['server']}"] + cfg_options = config.get("options") + if cfg_options: + if not isinstance(cfg_options, list): + raise RenderError("Expected [nfs_config.options] to be a list for [nfs] type") + + tracked_keys: set[str] = set() + disallowed_opts = ["addr"] + for opt in cfg_options: + if not isinstance(opt, str): + raise RenderError("Options for [nfs] type must be a list of strings.") + + key = opt.split("=")[0] + if key in tracked_keys: + raise RenderError(f"Option [{key}] already added for [nfs] type.") + if key in disallowed_opts: + raise RenderError(f"Option [{key}] is not allowed for [nfs] type.") + opts.append(opt) + tracked_keys.add(key) + + opts.sort() + + path = valid_fs_path_or_raise(config["path"].rstrip("/")) + self.volume_spec = { + "driver_opts": { + "type": "nfs", + "device": f":{path}", + "o": f"{','.join([escape_dollar(opt) for opt in opts])}", + }, + } + + def get(self): + return self.volume_spec + + +class CifsVolume: + def __init__(self, render_instance: "Render", config: "IxStorageCifsConfig"): + self._render_instance = render_instance + self.volume_spec: dict = {} + + if not config: + raise RenderError("Expected [cifs_config] to be set for [cifs] type") + + required_keys = ["server", "path", "username", "password"] + for key in required_keys: + if not config.get(key): + raise RenderError(f"Expected [{key}] to be set for [cifs] type") + + opts = [ + "noperm", + f"user={config['username']}", + f"password={config['password']}", + ] + + domain = config.get("domain") + if domain: + opts.append(f"domain={domain}") + + cfg_options = config.get("options") + if cfg_options: + if not isinstance(cfg_options, list): + raise RenderError("Expected [cifs_config.options] to be a list for [cifs] type") + + tracked_keys: set[str] = set() + disallowed_opts = ["user", "password", "domain", "noperm"] + for opt in cfg_options: + if not isinstance(opt, str): + raise RenderError("Options for [cifs] type must be a list of strings.") + + key = opt.split("=")[0] + if key in tracked_keys: + raise RenderError(f"Option [{key}] already added for [cifs] type.") + if key in disallowed_opts: + raise RenderError(f"Option [{key}] is not allowed for [cifs] type.") + for disallowed in disallowed_opts: + if key == disallowed: + raise RenderError(f"Option [{key}] is not allowed for [cifs] type.") + opts.append(opt) + tracked_keys.add(key) + opts.sort() + + server = config["server"].lstrip("/") + path = config["path"].strip("/") + path = valid_fs_path_or_raise("/" + path).lstrip("/") + + self.volume_spec = { + "driver_opts": { + "type": "cifs", + "device": f"//{server}/{path}", + "o": f"{','.join([escape_dollar(opt) for opt in opts])}", + }, + } + + def get(self): + return self.volume_spec + + +class DockerVolume: + def __init__(self, render_instance: "Render", config: "IxStorageVolumeConfig"): + self._render_instance = render_instance + self.volume_spec: dict = {} + + def get(self): + return self.volume_spec diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/volumes.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/volumes.py new file mode 100644 index 0000000000..e6925a402f --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_0_31/volumes.py @@ -0,0 +1,66 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + + +try: + from .error import RenderError + from .storage import IxStorageVolumeLikeConfigs + from .volume_types import NfsVolume, CifsVolume, DockerVolume +except ImportError: + from error import RenderError + from storage import IxStorageVolumeLikeConfigs + from volume_types import NfsVolume, CifsVolume, DockerVolume + + +class Volumes: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volumes: dict[str, Volume] = {} + + def add_volume( + self, + source: str, + storage_type: str, + config: "IxStorageVolumeLikeConfigs", + ): + # This method can be called many times from the volume mounts + # Only add the volume if it is not already added, but dont raise an error + if source == "": + raise RenderError(f"Volume source [{source}] cannot be empty") + + if source in self._volumes: + return + + self._volumes[source] = Volume(self._render_instance, storage_type, config) + + def has_volumes(self) -> bool: + return bool(self._volumes) + + def render(self): + return {name: v.render() for name, v in sorted(self._volumes.items()) if v.render() is not None} + + +class Volume: + def __init__( + self, + render_instance: "Render", + storage_type: str, + config: "IxStorageVolumeLikeConfigs", + ): + self._render_instance = render_instance + self.volume_spec: dict | None = {} + + match storage_type: + case "nfs": + self.volume_spec = NfsVolume(self._render_instance, config).get() # type: ignore + case "cifs": + self.volume_spec = CifsVolume(self._render_instance, config).get() # type: ignore + case "volume" | "temporary": + self.volume_spec = DockerVolume(self._render_instance, config).get() # type: ignore + case _: + self.volume_spec = None + + def render(self): + return self.volume_spec From 37fd0847a6cf3aee860931c972ab05d755aecf2e Mon Sep 17 00:00:00 2001 From: Stavros Kois Date: Tue, 3 Dec 2024 15:01:44 +0200 Subject: [PATCH 08/19] fix descv --- ix-dev/community/wordpress/questions.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ix-dev/community/wordpress/questions.yaml b/ix-dev/community/wordpress/questions.yaml index 6cde7c56b1..79f566a842 100644 --- a/ix-dev/community/wordpress/questions.yaml +++ b/ix-dev/community/wordpress/questions.yaml @@ -6,7 +6,7 @@ groups: - name: Storage Configuration description: Configure Storage for Wordpress - name: Labels Configuration - description: Configure Labels for Zerotier + description: Configure Labels for Wordpress - name: Resources Configuration description: Configure Resources for Wordpress From 12831615ce69c0551da796eb2ea2e20b701a1ec7 Mon Sep 17 00:00:00 2001 From: Stavros Kois Date: Tue, 3 Dec 2024 17:58:55 +0200 Subject: [PATCH 09/19] syncthing: use v2 --- ix-dev/enterprise/syncthing/app.yaml | 6 +- ix-dev/enterprise/syncthing/questions.yaml | 39 +- .../syncthing/templates/docker-compose.yaml | 261 +++---- .../library/base_v2_0_30/__init__.py | 0 .../templates/library/base_v2_0_30/configs.py | 86 +++ .../library/base_v2_0_30/container.py | 317 +++++++++ .../templates/library/base_v2_0_30/depends.py | 34 + .../templates/library/base_v2_0_30/deploy.py | 24 + .../templates/library/base_v2_0_30/deps.py | 454 ++++++++++++ .../templates/library/base_v2_0_30/device.py | 31 + .../templates/library/base_v2_0_30/devices.py | 66 ++ .../templates/library/base_v2_0_30/dns.py | 79 +++ .../library/base_v2_0_30/environment.py | 109 +++ .../templates/library/base_v2_0_30/error.py | 4 + .../library/base_v2_0_30/formatter.py | 26 + .../library/base_v2_0_30/functions.py | 142 ++++ .../library/base_v2_0_30/healthcheck.py | 193 +++++ .../templates/library/base_v2_0_30/labels.py | 37 + .../templates/library/base_v2_0_30/notes.py | 70 ++ .../templates/library/base_v2_0_30/portal.py | 22 + .../templates/library/base_v2_0_30/portals.py | 28 + .../templates/library/base_v2_0_30/ports.py | 68 ++ .../templates/library/base_v2_0_30/render.py | 89 +++ .../library/base_v2_0_30/resources.py | 115 +++ .../templates/library/base_v2_0_30/restart.py | 25 + .../templates/library/base_v2_0_30/storage.py | 116 +++ .../templates/library/base_v2_0_30/sysctls.py | 38 + .../library/base_v2_0_30/tests/__init__.py | 0 .../base_v2_0_30/tests/test_build_image.py | 49 ++ .../base_v2_0_30/tests/test_configs.py | 63 ++ .../base_v2_0_30/tests/test_container.py | 324 +++++++++ .../base_v2_0_30/tests/test_depends.py | 54 ++ .../library/base_v2_0_30/tests/test_deps.py | 380 ++++++++++ .../library/base_v2_0_30/tests/test_device.py | 121 ++++ .../library/base_v2_0_30/tests/test_dns.py | 64 ++ .../base_v2_0_30/tests/test_environment.py | 184 +++++ .../base_v2_0_30/tests/test_formatter.py | 13 + .../base_v2_0_30/tests/test_functions.py | 82 +++ .../base_v2_0_30/tests/test_healthcheck.py | 187 +++++ .../library/base_v2_0_30/tests/test_labels.py | 88 +++ .../library/base_v2_0_30/tests/test_notes.py | 213 ++++++ .../library/base_v2_0_30/tests/test_portal.py | 75 ++ .../library/base_v2_0_30/tests/test_ports.py | 110 +++ .../library/base_v2_0_30/tests/test_render.py | 37 + .../base_v2_0_30/tests/test_resources.py | 140 ++++ .../base_v2_0_30/tests/test_restart.py | 57 ++ .../base_v2_0_30/tests/test_sysctls.py | 62 ++ .../base_v2_0_30/tests/test_volumes.py | 666 ++++++++++++++++++ .../library/base_v2_0_30/validations.py | 227 ++++++ .../library/base_v2_0_30/volume_mount.py | 92 +++ .../base_v2_0_30/volume_mount_types.py | 72 ++ .../library/base_v2_0_30/volume_sources.py | 106 +++ .../library/base_v2_0_30/volume_types.py | 133 ++++ .../templates/library/base_v2_0_30/volumes.py | 66 ++ .../syncthing/templates/macros/setup.sh.jinja | 45 ++ .../templates/test_values/basic-values.yaml | 43 +- .../templates/test_values/https-values.yaml | 23 +- 57 files changed, 6056 insertions(+), 199 deletions(-) create mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/__init__.py create mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/configs.py create mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/container.py create mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/depends.py create mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/deploy.py create mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/deps.py create mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/device.py create mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/devices.py create mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/dns.py create mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/environment.py create mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/error.py create mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/formatter.py create mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/functions.py create mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/healthcheck.py create mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/labels.py create mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/notes.py create mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/portal.py create mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/portals.py create mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/ports.py create mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/render.py create mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/resources.py create mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/restart.py create mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/storage.py create mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/sysctls.py create mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/__init__.py create mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_build_image.py create mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_configs.py create mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_container.py create mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_depends.py create mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_deps.py create mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_device.py create mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_dns.py create mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_environment.py create mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_formatter.py create mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_functions.py create mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_healthcheck.py create mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_labels.py create mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_notes.py create mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_portal.py create mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_ports.py create mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_render.py create mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_resources.py create mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_restart.py create mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_sysctls.py create mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_volumes.py create mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/validations.py create mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/volume_mount.py create mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/volume_mount_types.py create mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/volume_sources.py create mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/volume_types.py create mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/volumes.py create mode 100644 ix-dev/enterprise/syncthing/templates/macros/setup.sh.jinja diff --git a/ix-dev/enterprise/syncthing/app.yaml b/ix-dev/enterprise/syncthing/app.yaml index 643ddad8ec..d6cc6ba0af 100644 --- a/ix-dev/enterprise/syncthing/app.yaml +++ b/ix-dev/enterprise/syncthing/app.yaml @@ -25,8 +25,8 @@ icon: https://media.sys.truenas.net/apps/syncthing/icons/icon.svg keywords: - sync - file-sharing -lib_version: 1.1.7 -lib_version_hash: d05e43e25b7dc1736be6cc1efa4b9255368aa346e3e7a4350a38440f29b73186 +lib_version: 2.0.30 +lib_version_hash: a87750c752394f7bf0eb461678564a9ef99bc4e4541787ef25e7be1095a6f879 maintainers: - email: dev@ixsystems.com name: truenas @@ -49,4 +49,4 @@ sources: - https://hub.docker.com/r/syncthing/syncthing title: Syncthing train: enterprise -version: 1.0.19 +version: 1.1.0 diff --git a/ix-dev/enterprise/syncthing/questions.yaml b/ix-dev/enterprise/syncthing/questions.yaml index be00299b84..276624345e 100644 --- a/ix-dev/enterprise/syncthing/questions.yaml +++ b/ix-dev/enterprise/syncthing/questions.yaml @@ -7,6 +7,8 @@ groups: description: Configure Network for Syncthing - name: Storage Configuration description: Configure Storage for Syncthing + - name: Labels Configuration + description: Configure Labels for Syncthing - name: Resources Configuration description: Configure Resources for Syncthing @@ -337,7 +339,42 @@ questions: description: The domain to use for the SMB share. schema: type: string - + - variable: labels + label: "" + group: Labels Configuration + schema: + type: list + default: [] + items: + - variable: label + label: Label + schema: + type: dict + attrs: + - variable: key + label: Key + schema: + type: string + required: true + - variable: value + label: Value + schema: + type: string + required: true + - variable: containers + label: Containers + description: Containers where the label should be applied + schema: + type: list + items: + - variable: container + label: Container + schema: + type: string + required: true + enum: + - value: syncthing + description: syncthing - variable: resources label: "" group: Resources Configuration diff --git a/ix-dev/enterprise/syncthing/templates/docker-compose.yaml b/ix-dev/enterprise/syncthing/templates/docker-compose.yaml index 84bdd0f1d0..871dec871f 100644 --- a/ix-dev/enterprise/syncthing/templates/docker-compose.yaml +++ b/ix-dev/enterprise/syncthing/templates/docker-compose.yaml @@ -1,183 +1,98 @@ -{# Stores Syncthing storage items that contains info for volumes, vol mounts, perms dirs and perms mounts #} -{% set storage_items = namespace(items=[]) %} -{# Stores the Syncthing container volume mounts #} -{% set volume_mounts = namespace(items=[]) %} -{# Stores the top level volumes #} -{% set volumes = namespace(items={}) %} - -{% do storage_items.items.append(ix_lib.base.storage.storage_item(data=dict(values.storage.home, **{"mount_path": values.consts.home_path}), values=values)) %} -{% do storage_items.items.append(ix_lib.base.storage.storage_item(data={"type":"anonymous", "mount_path": "/tmp"})) %} -{% for store in values.storage.additional_storage %} - {% if store.type == "cifs" and store.cifs_config.migration_mode %} - {% do store.update({"read_only": true}) %} - {% do store.cifs_config.update({"options": ["noperm", "cifsacl", "vers=3.0"]}) %} - {% endif %} - {% do storage_items.items.append(ix_lib.base.storage.storage_item(data=store, values=values)) %} -{% else %} - {% do ix_lib.base.utils.throw_error("Expected at least one storage item to be set for Syncthing") %} -{% endfor %} +{% from "macros/setup.sh.jinja" import setup_script %} +{% set tpl = ix_lib.base.render.Render(values) %} -{# Add each item to the above lists #} -{% for item in storage_items.items %} - {% if item.vol and volumes.items.update(item.vol) %}{% endif %} - {% if item.vol_mount and volume_mounts.items.append(item.vol_mount) %}{% endif %} -{% endfor %} +{% set st = values.consts.settings %} +{% set settings = [ + {"cmd": "options announce-lanaddresses", "value": 1 if st.announce_lan_addresses else 0, "quote": true}, + {"cmd": "options global-ann-enabled", "value": 1 if st.global_discovery else 0, "quote": true}, + {"cmd": "options local-ann-enabled", "value": 1 if st.local_discovery else 0, "quote": true}, + {"cmd": "options natenabled", "value": 1 if st.nat_traversal else 0, "quote": true}, + {"cmd": "options relays-enabled", "value": 1 if st.relaying else 0, "quote": true}, + {"cmd": "options uraccepted", "value": 1 if st.telemetry else -1, "quote": true}, + {"cmd": "options auto-upgrade-intervalh", "value": st.auto_upgrade_intervalh, "quote": true}, + {"cmd": "defaults folder xattr-filter max-total-size", "value": st.xattr_filter_max_total_size, "quote": false}, + {"cmd": "defaults folder xattr-filter max-single-entry-size", "value": st.xattr_filter_max_single_entry_size, "quote": true}, + {"cmd": "defaults folder send-ownership", "value": 1 if st.send_ownership else 0, "quote": false}, + {"cmd": "defaults folder sync-ownership", "value": 1 if st.sync_ownership else 0, "quote": false}, + {"cmd": "defaults folder send-xattrs", "value": 1 if st.send_xattrs else 0, "quote": false}, + {"cmd": "defaults folder sync-xattrs", "value": 1 if st.sync_xattrs else 0, "quote": false}, + {"cmd": "defaults folder ignore-perms", "value": 1 if st.ignore_perms else 0, "quote": false}, + {"cmd": "defaults folder path", "value": st.path, "quote": true}, +] %} -{# Configs #} -configs: - logo-horizontal-svg: - content: {{ values.consts.logo_horizontal_svg | tojson }} -{% if values.network.certificate_id %} - private: - content: {{ values.ix_certificates[values.network.certificate_id].privatekey | tojson }} - public: - content: {{ values.ix_certificates[values.network.certificate_id].certificate | tojson }} -{% endif %} +{% set c1 = tpl.add_container(values.consts.syncthing_container_name, "image") %} +{% set config = tpl.add_container(values.consts.config_container_name, "image") %} + +{% do c1.depends.add_dependency(values.consts.config_container_name, "service_completed_successfully") %} +{% do config.restart.set_policy("on-failure", 1) %} +{% do config.remove_devices() %} +{% do config.deploy.resources.set_profile("medium") %} +{% do config.configs.add("setup.sh", setup_script(values, settings), "/setup.sh", "0755") %} +{% do config.set_entrypoint(["/setup.sh"]) %} -{% set caps = ix_lib.base.security.get_caps(add=["CHOWN", "DAC_OVERRIDE", "FOWNER", "SETGID", "SETUID", "SETFCAP", "SETPCAP", "SYS_ADMIN"]) %} -{% set app_env = { +{% do c1.set_user(0, 0) %} +{% do config.set_user(0, 0) %} + +{% do c1.remove_security_opt("no-new-privileges") %} +{% do config.remove_security_opt("no-new-privileges") %} + +{% set caps = ["CHOWN", "DAC_OVERRIDE", "FOWNER", "SETGID", "SETUID", "SETFCAP", "SETPCAP", "SYS_ADMIN"] %} +{% do c1.add_caps(caps) %} +{% do config.add_caps(caps + ["KILL"]) %} + +{% do c1.healthcheck.set_test("wget", {"port": values.network.web_port, "path": "/rest/noauth/health"}) %} +{% do config.healthcheck.disable() %} + +{% set envs = { "PCAP": ["cap_sys_admin", "cap_chown", "cap_dac_override", "cap_fowner"]|join(",") + "+ep", "STNOUPGRADE": true, "STGUIADDRESS": "0.0.0.0:%d" | format(values.network.web_port), "STGUIASSETS": "/var/truenas/assets/gui", } %} -{# Containers #} -services: - {{ values.consts.config_container_name }}: - image: {{ ix_lib.base.utils.get_image(images=values.images, name="image") }} - user: "0:0" - deploy: - resources: {{ ix_lib.base.resources.resources(values.resources) | tojson }} - devices: {{ ix_lib.base.resources.get_devices(values.resources) | tojson }} - configs: - - source: logo-horizontal-svg - target: {{ values.consts.logo_horizontal_svg_path }} - {% if values.network.certificate_id %} - - source: private - target: {{ "%s/config/https-key.pem" | format(values.consts.home_path) }} - - source: public - target: {{ "%s/config/https-cert.pem" | format(values.consts.home_path) }} - {% endif %} - {% set config_caps = ix_lib.base.security.get_caps(add=caps.add + ["KILL"]) %} - cap_add: {{ config_caps.add | tojson }} - cap_drop: {{ config_caps.drop | tojson }} - security_opt: {{ ix_lib.base.security.get_sec_opts(remove=["no-new-privileges"]) | tojson }} - healthcheck: - disable: true - entrypoint: - - /bin/sh - {% set config_dir = "%s/config"|format(values.consts.home_path) %} - {% set cli = "syncthing cli --home %s config"|format(config_dir) %} - {% set st = values.consts.settings %} - {% set settings = [ - {"cmd": "options announce-lanaddresses", "value": 1 if st.announce_lan_addresses else 0, "quote": true}, - {"cmd": "options global-ann-enabled", "value": 1 if st.global_discovery else 0, "quote": true}, - {"cmd": "options local-ann-enabled", "value": 1 if st.local_discovery else 0, "quote": true}, - {"cmd": "options natenabled", "value": 1 if st.nat_traversal else 0, "quote": true}, - {"cmd": "options relays-enabled", "value": 1 if st.relaying else 0, "quote": true}, - {"cmd": "options uraccepted", "value": 1 if st.telemetry else -1, "quote": true}, - {"cmd": "options auto-upgrade-intervalh", "value": st.auto_upgrade_intervalh, "quote": true}, - {"cmd": "defaults folder xattr-filter max-total-size", "value": st.xattr_filter_max_total_size, "quote": false}, - {"cmd": "defaults folder xattr-filter max-single-entry-size", "value": st.xattr_filter_max_single_entry_size, "quote": true}, - {"cmd": "defaults folder send-ownership", "value": 1 if st.send_ownership else 0, "quote": false}, - {"cmd": "defaults folder sync-ownership", "value": 1 if st.sync_ownership else 0, "quote": false}, - {"cmd": "defaults folder send-xattrs", "value": 1 if st.send_xattrs else 0, "quote": false}, - {"cmd": "defaults folder sync-xattrs", "value": 1 if st.sync_xattrs else 0, "quote": false}, - {"cmd": "defaults folder ignore-perms", "value": 1 if st.ignore_perms else 0, "quote": false}, - {"cmd": "defaults folder path", "value": st.path, "quote": true}, - ] %} - command: - - -c - - | - set -e - trap cleanup EXIT TERM - cleanup() { - echo "Gracefully stopping Syncthing..." - if kill -0 $$SYNCTHING_PID > /dev/null 2>&1; then - kill -SIGTERM $$SYNCTHING_PID - wait $$SYNCTHING_PID - fi - echo "Syncthing stopped." - } - try_for() { - local max_tries=$$1 - local sleep_time=$$2 - local cmd=$$3 - tries=0 - until eval "$$cmd"; do - [ $$tries -ge $$max_tries ] && return 1 - tries=$$((tries+1)) - sleep $$sleep_time - done - } - - echo "Starting Syncthing in the background" - /bin/entrypoint.sh /bin/syncthing & - SYNCTHING_PID=$$! - echo "Syncthing started with PID [$$SYNCTHING_PID]" - echo "Waiting for Syncthing to be ready..." - - try_for 15 2 "[ -f '{{ config_dir }}/config.xml' ]" || { echo "Syncthing did not become ready in time. Exiting..."; exit 1; } - try_for 15 2 "curl --silent --output /dev/null http://127.0.0.1:{{ values.network.web_port }}/rest/noauth/health" || { echo "Syncthing did not become ready in time. Exiting..."; exit 1; } - echo "Syncthing is ready." - - {% for cfg in settings %} - echo 'Using subcommand [{{ cfg.cmd }}] to set value [{{ '\"%s\"' | format(cfg.value) if cfg.quote else cfg.value }}]' - {{ cli }} {{ cfg.cmd }} set -- {{ '"%s"' | format(cfg.value) if cfg.quote else cfg.value }} || { echo "Failed to apply. Exiting..."; exit 1; } - {% endfor %} - - echo "Gracefully stopping Syncthing..." - kill -SIGTERM $$SYNCTHING_PID - wait $$SYNCTHING_PID - echo "Syncthing stopped." - - environment: {{ ix_lib.base.environment.envs(app=app_env, user=values.syncthing.additional_envs, values=values) | tojson }} - volumes: {{ volume_mounts.items | tojson }} - {{ values.consts.syncthing_container_name }}: - image: {{ ix_lib.base.utils.get_image(images=values.images, name="image") }} - user: "0:0" - restart: unless-stopped - deploy: - resources: {{ ix_lib.base.resources.resources(values.resources) | tojson }} - devices: {{ ix_lib.base.resources.get_devices(values.resources) | tojson }} - cap_add: {{ caps.add | tojson }} - cap_drop: {{ caps.drop | tojson }} - security_opt: {{ ix_lib.base.security.get_sec_opts(remove=["no-new-privileges"]) | tojson }} - {% if values.network.host_network %} - network_mode: host - {% endif %} - depends_on: - {{ values.consts.config_container_name }}: - condition: service_completed_successfully - {% if values.network.dns_opts %} - dns_opt: {{ ix_lib.base.network.dns_opts(values.network.dns_opts) | tojson }} - {% endif %} - configs: - - source: logo-horizontal-svg - target: {{ values.consts.logo_horizontal_svg_path }} - {% if values.network.certificate_id %} - - source: private - target: {{ "%s/config/https-key.pem" | format(values.consts.home_path) }} - - source: public - target: {{ "%s/config/https-cert.pem" | format(values.consts.home_path) }} - {% endif %} - {% set test = ix_lib.base.healthchecks.wget_test(port=values.network.web_port, path="/rest/noauth/health") %} - healthcheck: {{ ix_lib.base.healthchecks.check_health(test) | tojson }} - volumes: {{ volume_mounts.items | tojson }} - environment: {{ ix_lib.base.environment.envs(app=app_env, user=values.syncthing.additional_envs, values=values) | tojson }} - {% if not values.network.host_network %} - ports: - - {{ ix_lib.base.ports.get_port(port={"target": values.network.web_port, "published": values.network.web_port}) | tojson }} - - {{ ix_lib.base.ports.get_port(port={"target": 22000, "published": values.network.tcp_port}) | tojson }} - - {{ ix_lib.base.ports.get_port(port={"target": 22000, "published": values.network.quic_port, "protocol": "udp"}) | tojson }} - {% if values.consts.settings.local_discovery %} - - {{ ix_lib.base.ports.get_port(port={"target": 27017 , "published": values.network.local_discover_port, "protocol": "udp"}) | tojson }} - {% endif %} - {% endif %} - -{% if volumes.items %} -volumes: {{ volumes.items | tojson }} + +{% for k, v in envs.items() %} + {% do c1.environment.add_env(k, v) %} + {% do config.environment.add_env(k, v) %} +{% endfor %} + +{% do c1.environment.add_user_envs(values.syncthing.additional_envs) %} +{% do config.environment.add_user_envs(values.syncthing.additional_envs) %} + +{% do c1.configs.add("logo-horizontal-svg", values.consts.logo_horizontal_svg, values.consts.logo_horizontal_svg_path) %} +{% do config.configs.add("logo-horizontal-svg", values.consts.logo_horizontal_svg, values.consts.logo_horizontal_svg_path) %} + +{% if values.network.certificate_id %} + {% set cert = values.ix_certificates[values.network.certificate_id] %} + {% do c1.configs.add("private", cert.certificate, "%s/config/https-key.pem" | format(values.consts.home_path)) %} + {% do c1.configs.add("public", cert.certificate, "%s/config/https-cert.pem" | format(values.consts.home_path)) %} + + {% do config.configs.add("private", cert.privatekey, "%s/config/https-key.pem" | format(values.consts.home_path)) %} + {% do config.configs.add("public", cert.certificate, "%s/config/https-cert.pem" | format(values.consts.home_path)) %} +{% endif %} + +{% do c1.ports.add_port(values.network.web_port, values.network.web_port) %} +{% if not values.network.host_network %} + {% do c1.ports.add_port(values.network.tcp_port, 22000) %} + {% do c1.ports.add_port(values.network.quic_port, 22000, {"protocol": "udp"}) %} + {% if values.consts.settings.local_discovery %} + {% do c1.ports.add_port(values.network.local_discover_port, 27017, {"protocol": "udp"}) %} + {% endif %} {% endif %} -x-portals: {{ ix_lib.base.metadata.get_portals([{"port": values.network.web_port, "scheme": "https" if values.network.certificate_id else "http"}]) | tojson }} -x-notes: {{ ix_lib.base.metadata.get_notes("Syncthing") | tojson }} +{% for store in values.storage.additional_storage %} + {% set new_store = tpl.funcs.copy_dict(store) %} + {% if new_store.type == "cifs" and new_store.cifs_config.migration_mode %} + {% do new_store.update({"read_only": true}) %} + {% do new_store.cifs_config.update({"options": ["cifsacl", "vers=3.0"]}) %} + {% endif %} + + {% do c1.add_storage(new_store.mount_path, new_store) %} + {% do config.add_storage(new_store.mount_path, new_store) %} +{% else %} + {% do tpl.funcs.fail("Expected at least one storage item to be set for Syncthing") %} +{% endfor %} + +{% set proto = "https" if values.network.certificate_id else "http" %} +{% do tpl.portals.add_portal({"port": values.network.web_port, "scheme": proto})%} + +{{ tpl.render() | tojson }} diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/__init__.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/configs.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/configs.py new file mode 100644 index 0000000000..b76f4b169c --- /dev/null +++ b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/configs.py @@ -0,0 +1,86 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class Configs: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._configs: dict[str, dict] = {} + + def add(self, name: str, data: str): + if not isinstance(data, str): + raise RenderError(f"Expected [data] to be a string, got [{type(data)}]") + + if name not in self._configs: + self._configs[name] = {"name": name, "data": data} + return + + if data == self._configs[name]["data"]: + return + + raise RenderError(f"Config [{name}] already added with different data") + + def has_configs(self): + return bool(self._configs) + + def render(self): + return { + c["name"]: {"content": escape_dollar(c["data"])} + for c in sorted(self._configs.values(), key=lambda c: c["name"]) + } + + +class ContainerConfigs: + def __init__(self, render_instance: "Render", configs: Configs): + self._render_instance = render_instance + self.top_level_configs: Configs = configs + self.container_configs: set[ContainerConfig] = set() + + def add(self, name: str, data: str, target: str, mode: str = ""): + self.top_level_configs.add(name, data) + + if target == "": + raise RenderError(f"Expected [target] to be set for config [{name}]") + if mode != "": + mode = valid_octal_mode_or_raise(mode) + + if target in [c.target for c in self.container_configs]: + raise RenderError(f"Target [{target}] already used for another config") + target = valid_fs_path_or_raise(target) + self.container_configs.add(ContainerConfig(self._render_instance, name, target, mode)) + + def has_configs(self): + return bool(self.container_configs) + + def render(self): + return [c.render() for c in sorted(self.container_configs, key=lambda c: c.source)] + + +class ContainerConfig: + def __init__(self, render_instance: "Render", source: str, target: str, mode: str): + self._render_instance = render_instance + self.source = source + self.target = target + self.mode = mode + + def render(self): + result: dict[str, str | int] = { + "source": self.source, + "target": self.target, + } + + if self.mode: + result["mode"] = int(self.mode, 8) + + return result diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/container.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/container.py new file mode 100644 index 0000000000..a95e76734c --- /dev/null +++ b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/container.py @@ -0,0 +1,317 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + "cap_drop": sorted(self._cap_drop), + "healthcheck": self.healthcheck.render(), + } + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/depends.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/depends.py new file mode 100644 index 0000000000..4e057cf085 --- /dev/null +++ b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/depends.py @@ -0,0 +1,34 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_depend_condition_or_raise +except ImportError: + from error import RenderError + from validations import valid_depend_condition_or_raise + + +class Depends: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._dependencies: dict[str, str] = {} + + def add_dependency(self, name: str, condition: str): + condition = valid_depend_condition_or_raise(condition) + if name in self._dependencies.keys(): + raise RenderError(f"Dependency [{name}] already added") + if name not in self._render_instance.container_names(): + raise RenderError( + f"Dependency [{name}] not found in defined containers. " + f"Available containers: [{', '.join(self._render_instance.container_names())}]" + ) + self._dependencies[name] = condition + + def has_dependencies(self): + return len(self._dependencies) > 0 + + def render(self): + return {d: {"condition": c} for d, c in self._dependencies.items()} diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/deploy.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/deploy.py new file mode 100644 index 0000000000..894dbc643b --- /dev/null +++ b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/deploy.py @@ -0,0 +1,24 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .resources import Resources +except ImportError: + from resources import Resources + + +class Deploy: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self.resources: Resources = Resources(self._render_instance) + + def has_deploy(self): + return self.resources.has_resources() + + def render(self): + if self.resources.has_resources(): + return {"resources": self.resources.render()} + + return {} diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/deps.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/deps.py new file mode 100644 index 0000000000..b3607fa6ab --- /dev/null +++ b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/deps.py @@ -0,0 +1,454 @@ +import os +import json +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import ( + valid_port_or_raise, + valid_fs_path_or_raise, + valid_octal_mode_or_raise, + valid_redis_password_or_raise, + ) +except ImportError: + from error import RenderError + from validations import ( + valid_port_or_raise, + valid_fs_path_or_raise, + valid_octal_mode_or_raise, + valid_redis_password_or_raise, + ) + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = os.path.join("/mnt/permission", identifier) + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + + c.add_storage("/var/lib/postgresql/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("POSTGRES_USER", config["user"]) + c.environment.add_env("POSTGRES_PASSWORD", config["password"]) + c.environment.add_env("POSTGRES_DB", config["database"]) + c.environment.add_env("POSTGRES_PORT", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def _get_port(self): + return self._config.get("port") or 5432 + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/device.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/device.py new file mode 100644 index 0000000000..bfe97097cb --- /dev/null +++ b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/device.py @@ -0,0 +1,31 @@ +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise, allowed_device_or_raise, valid_cgroup_perm_or_raise +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise, allowed_device_or_raise, valid_cgroup_perm_or_raise + + +class Device: + def __init__(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + hd = valid_fs_path_or_raise(host_device.rstrip("/")) + cd = valid_fs_path_or_raise(container_device.rstrip("/")) + if not hd or not cd: + raise RenderError( + "Expected [host_device] and [container_device] to be set. " + f"Got host_device [{host_device}] and container_device [{container_device}]" + ) + + cgroup_perm = valid_cgroup_perm_or_raise(cgroup_perm) + if not allow_disallowed: + hd = allowed_device_or_raise(hd) + + self.cgroup_perm: str = cgroup_perm + self.host_device: str = hd + self.container_device: str = cd + + def render(self): + result = f"{self.host_device}:{self.container_device}" + if self.cgroup_perm: + result += f":{self.cgroup_perm}" + return result diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/devices.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/devices.py new file mode 100644 index 0000000000..ae22c79d2e --- /dev/null +++ b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/devices.py @@ -0,0 +1,66 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/dns.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/dns.py new file mode 100644 index 0000000000..d3ae7b19fa --- /dev/null +++ b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/dns.py @@ -0,0 +1,79 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import allowed_dns_opt_or_raise +except ImportError: + from error import RenderError + from validations import allowed_dns_opt_or_raise + + +class Dns: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._dns_options: set[str] = set() + self._dns_searches: set[str] = set() + self._dns_nameservers: set[str] = set() + + self._auto_add_dns_opts_from_values() + self._auto_add_dns_searches_from_values() + self._auto_add_dns_nameservers_from_values() + + def _get_dns_opt_keys(self): + return [self._get_key_from_opt(opt) for opt in self._dns_options] + + def _get_key_from_opt(self, opt): + return opt.split(":")[0] + + def _auto_add_dns_opts_from_values(self): + values = self._render_instance.values + for dns_opt in values.get("network", {}).get("dns_opts", []): + self.add_dns_opt(dns_opt) + + def _auto_add_dns_searches_from_values(self): + values = self._render_instance.values + for dns_search in values.get("network", {}).get("dns_searches", []): + self.add_dns_search(dns_search) + + def _auto_add_dns_nameservers_from_values(self): + values = self._render_instance.values + for dns_nameserver in values.get("network", {}).get("dns_nameservers", []): + self.add_dns_nameserver(dns_nameserver) + + def add_dns_search(self, dns_search): + if dns_search in self._dns_searches: + raise RenderError(f"DNS Search [{dns_search}] already added") + self._dns_searches.add(dns_search) + + def add_dns_nameserver(self, dns_nameserver): + if dns_nameserver in self._dns_nameservers: + raise RenderError(f"DNS Nameserver [{dns_nameserver}] already added") + self._dns_nameservers.add(dns_nameserver) + + def add_dns_opt(self, dns_opt): + # eg attempts:3 + key = allowed_dns_opt_or_raise(self._get_key_from_opt(dns_opt)) + if key in self._get_dns_opt_keys(): + raise RenderError(f"DNS Option [{key}] already added") + self._dns_options.add(dns_opt) + + def has_dns_opts(self): + return len(self._dns_options) > 0 + + def has_dns_searches(self): + return len(self._dns_searches) > 0 + + def has_dns_nameservers(self): + return len(self._dns_nameservers) > 0 + + def render_dns_searches(self): + return sorted(self._dns_searches) + + def render_dns_opts(self): + return sorted(self._dns_options) + + def render_dns_nameservers(self): + return sorted(self._dns_nameservers) diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/environment.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/environment.py new file mode 100644 index 0000000000..850a3afd8e --- /dev/null +++ b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/environment.py @@ -0,0 +1,109 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/error.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/error.py new file mode 100644 index 0000000000..aef48d3b02 --- /dev/null +++ b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/error.py @@ -0,0 +1,4 @@ +class RenderError(Exception): + """Base class for exceptions in this module.""" + + pass diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/formatter.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/formatter.py new file mode 100644 index 0000000000..24e882f47a --- /dev/null +++ b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/formatter.py @@ -0,0 +1,26 @@ +import json +import hashlib + + +def escape_dollar(text: str) -> str: + return text.replace("$", "$$") + + +def get_hashed_name_for_volume(prefix: str, config: dict): + config_hash = hashlib.sha256(json.dumps(config).encode("utf-8")).hexdigest() + return f"{prefix}_{config_hash}" + + +def get_hash_with_prefix(prefix: str, data: str): + return f"{prefix}_{hashlib.sha256(data.encode('utf-8')).hexdigest()}" + + +def merge_dicts_no_overwrite(dict1, dict2): + overlapping_keys = dict1.keys() & dict2.keys() + if overlapping_keys: + raise ValueError(f"Merging of dicts failed. Overlapping keys: {overlapping_keys}") + return {**dict1, **dict2} + + +def get_image_with_hashed_data(image: str, data: str): + return get_hash_with_prefix(f"ix-{image}", data) diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/functions.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/functions.py new file mode 100644 index 0000000000..b0c834ab4b --- /dev/null +++ b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/functions.py @@ -0,0 +1,142 @@ +import re +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return dict.copy() + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + } diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/healthcheck.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/healthcheck.py new file mode 100644 index 0000000000..36ae5d90aa --- /dev/null +++ b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/healthcheck.py @@ -0,0 +1,193 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def render(self): + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/labels.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/labels.py new file mode 100644 index 0000000000..f1e667ba00 --- /dev/null +++ b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/labels.py @@ -0,0 +1,37 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar +except ImportError: + from error import RenderError + from formatter import escape_dollar + + +class Labels: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._labels: dict[str, str] = {} + + def add_label(self, key: str, value: str): + if not key: + raise RenderError("Labels must have a key") + + if key.startswith("com.docker.compose"): + raise RenderError(f"Label [{key}] cannot start with [com.docker.compose] as it is reserved") + + if key in self._labels.keys(): + raise RenderError(f"Label [{key}] already added") + + self._labels[key] = escape_dollar(str(value)) + + def has_labels(self) -> bool: + return bool(self._labels) + + def render(self) -> dict[str, str]: + if not self.has_labels(): + return {} + return {label: value for label, value in sorted(self._labels.items())} diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/notes.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/notes.py new file mode 100644 index 0000000000..4adc50c3d8 --- /dev/null +++ b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/notes.py @@ -0,0 +1,70 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + + +class Notes: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._app_name: str = "" + self._warnings: list[str] = [] + self._deprecations: list[str] = [] + self._header: str = "" + self._body: str = "" + self._footer: str = "" + + self._auto_set_app_name() + self._auto_set_header() + self._auto_set_footer() + + def _auto_set_app_name(self): + app_name = self._render_instance.values.get("ix_context", {}).get("app_metadata", {}).get("name", "") + self._app_name = app_name or "" + + def _auto_set_header(self): + head = "# Welcome to TrueNAS SCALE\n\n" + head += f"Thank you for installing {self._app_name}!\n\n" + self._header = head + + def _auto_set_footer(self): + footer = "## Documentation\n\n" + footer += f"Documentation for {self._app_name} can be found at https://www.truenas.com/docs.\n\n" + footer += "## Bug reports\n\n" + footer += "If you find a bug in this app, please file an issue at\n" + footer += "https://ixsystems.atlassian.net or https://github.com/truenas/apps\n\n" + footer += "## Feature requests or improvements\n\n" + footer += "If you find a feature request for this app, please file an issue at\n" + footer += "https://ixsystems.atlassian.net or https://github.com/truenas/apps\n" + self._footer = footer + + def add_warning(self, warning: str): + self._warnings.append(warning) + + def add_deprecation(self, deprecation: str): + self._deprecations.append(deprecation) + + def set_body(self, body: str): + self._body = body + + def render(self): + result = self._header + + if self._warnings: + result += "## Warnings\n\n" + for warning in self._warnings: + result += f"- {warning}\n" + result += "\n" + + if self._deprecations: + result += "## Deprecations\n\n" + for deprecation in self._deprecations: + result += f"- {deprecation}\n" + result += "\n" + + if self._body: + result += self._body.strip() + "\n\n" + + result += self._footer + + return result diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/portal.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/portal.py new file mode 100644 index 0000000000..cf47163439 --- /dev/null +++ b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/portal.py @@ -0,0 +1,22 @@ +try: + from .validations import valid_portal_scheme_or_raise, valid_http_path_or_raise, valid_port_or_raise +except ImportError: + from validations import valid_portal_scheme_or_raise, valid_http_path_or_raise, valid_port_or_raise + + +class Portal: + def __init__(self, name: str, config: dict): + self._name = name + self._scheme = valid_portal_scheme_or_raise(config.get("scheme", "http")) + self._host = config.get("host", "0.0.0.0") or "0.0.0.0" + self._port = valid_port_or_raise(config.get("port", 0)) + self._path = valid_http_path_or_raise(config.get("path", "/")) + + def render(self): + return { + "name": self._name, + "scheme": self._scheme, + "host": self._host, + "port": self._port, + "path": self._path, + } diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/portals.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/portals.py new file mode 100644 index 0000000000..e106d231e6 --- /dev/null +++ b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/portals.py @@ -0,0 +1,28 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .portal import Portal +except ImportError: + from error import RenderError + from portal import Portal + + +class Portals: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._portals: set[Portal] = set() + + def add_portal(self, config: dict): + name = config.get("name", "Web UI") + + if name in [p._name for p in self._portals]: + raise RenderError(f"Portal [{name}] already added") + + self._portals.add(Portal(name, config)) + + def render(self): + return [p.render() for _, p in sorted([(p._name, p) for p in self._portals])] diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/ports.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/ports.py new file mode 100644 index 0000000000..f11e1481b4 --- /dev/null +++ b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/ports.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import ( + valid_port_or_raise, + valid_port_protocol_or_raise, + valid_port_mode_or_raise, + valid_ip_or_raise, + ) +except ImportError: + from error import RenderError + from validations import ( + valid_port_or_raise, + valid_port_protocol_or_raise, + valid_port_mode_or_raise, + valid_ip_or_raise, + ) + + +class Ports: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._ports: dict[str, dict] = {} + + def add_port(self, host_port: int, container_port: int, config: dict | None = None): + config = config or {} + host_port = valid_port_or_raise(host_port) + container_port = valid_port_or_raise(container_port) + proto = valid_port_protocol_or_raise(config.get("protocol", "tcp")) + mode = valid_port_mode_or_raise(config.get("mode", "ingress")) + host_ip = valid_ip_or_raise(config.get("host_ip", "0.0.0.0")) + + key = f"{host_port}_{host_ip}_{proto}" + if key in self._ports.keys(): + raise RenderError(f"Port [{host_port}/{proto}] already added for [{host_ip}]") + + if host_ip != "0.0.0.0": + # If the port we are adding is not going to use 0.0.0.0 + # Make sure that we don't have already added that port/proto to 0.0.0.0 + search_key = f"{host_port}_0.0.0.0_{proto}" + if search_key in self._ports.keys(): + raise RenderError(f"Cannot bind port [{host_port}/{proto}] to [{host_ip}], already bound to [0.0.0.0]") + elif host_ip == "0.0.0.0": + # If the port we are adding is going to use 0.0.0.0 + # Make sure that we don't have already added that port/proto to a specific ip + for p in self._ports.values(): + if p["published"] == host_port and p["protocol"] == proto: + raise RenderError( + f"Cannot bind port [{host_port}/{proto}] to [{host_ip}], already bound to [{p['host_ip']}]" + ) + + self._ports[key] = { + "published": host_port, + "target": container_port, + "protocol": proto, + "mode": mode, + "host_ip": host_ip, + } + + def has_ports(self): + return len(self._ports) > 0 + + def render(self): + return [config for _, config in sorted(self._ports.items())] diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/render.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/render.py new file mode 100644 index 0000000000..9d8fcc28d5 --- /dev/null +++ b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/render.py @@ -0,0 +1,89 @@ +import copy + +try: + from .configs import Configs + from .container import Container + from .deps import Deps + from .error import RenderError + from .functions import Functions + from .notes import Notes + from .portals import Portals + from .volumes import Volumes +except ImportError: + from configs import Configs + from container import Container + from deps import Deps + from error import RenderError + from functions import Functions + from notes import Notes + from portals import Portals + from volumes import Volumes + + +class Render(object): + def __init__(self, values): + self._containers: dict[str, Container] = {} + self.values = values + self._add_images_internal_use() + # Make a copy after we inject the images + self._original_values: dict = copy.deepcopy(self.values) + + self.deps: Deps = Deps(self) + + self.configs = Configs(render_instance=self) + self.funcs = Functions(render_instance=self).func_map() + self.portals: Portals = Portals(render_instance=self) + self.notes: Notes = Notes(render_instance=self) + self.volumes = Volumes(render_instance=self) + + def _add_images_internal_use(self): + if not self.values.get("images"): + self.values["images"] = {} + + if "python_permissions_image" not in self.values["images"]: + self.values["images"]["python_permissions_image"] = {"repository": "python", "tag": "3.13.0-slim-bookworm"} + + def container_names(self): + return list(self._containers.keys()) + + def add_container(self, name: str, image: str): + name = name.strip() + if not name: + raise RenderError("Container name cannot be empty") + container = Container(self, name, image) + if name in self._containers: + raise RenderError(f"Container {name} already exists.") + self._containers[name] = container + return container + + def render(self): + if self.values != self._original_values: + raise RenderError("Values have been modified since the renderer was created.") + + if not self._containers: + raise RenderError("No containers added.") + + result: dict = { + "x-notes": self.notes.render(), + "x-portals": self.portals.render(), + "services": {c._name: c.render() for c in self._containers.values()}, + } + + # Make sure that after services are rendered + # there are no labels that target a non-existent container + # This is to prevent typos + for label in self.values.get("labels", []): + for c in label.get("containers", []): + if c not in self.container_names(): + raise RenderError(f"Label [{label['key']}] references container [{c}] which does not exist") + + if self.volumes.has_volumes(): + result["volumes"] = self.volumes.render() + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + # if self.networks: + # result["networks"] = {...} + + return result diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/resources.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/resources.py new file mode 100644 index 0000000000..733f43bb6f --- /dev/null +++ b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/resources.py @@ -0,0 +1,115 @@ +import re +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +DEFAULT_CPUS = 2.0 +DEFAULT_MEMORY = 4096 + + +class Resources: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._limits: dict = {} + self._reservations: dict = {} + self._nvidia_ids: set[str] = set() + self._auto_add_cpu_from_values() + self._auto_add_memory_from_values() + self._auto_add_gpus_from_values() + + def _set_cpu(self, cpus: Any): + c = str(cpus) + if not re.match(r"^[1-9][0-9]*(\.[0-9]+)?$", c): + raise RenderError(f"Expected cpus to be a number or a float (minimum 1.0), got [{cpus}]") + self._limits.update({"cpus": c}) + + def _set_memory(self, memory: Any): + m = str(memory) + if not re.match(r"^[1-9][0-9]*$", m): + raise RenderError(f"Expected memory to be a number, got [{memory}]") + self._limits.update({"memory": f"{m}M"}) + + def _auto_add_cpu_from_values(self): + resources = self._render_instance.values.get("resources", {}) + self._set_cpu(resources.get("limits", {}).get("cpus", DEFAULT_CPUS)) + + def _auto_add_memory_from_values(self): + resources = self._render_instance.values.get("resources", {}) + self._set_memory(resources.get("limits", {}).get("memory", DEFAULT_MEMORY)) + + def _auto_add_gpus_from_values(self): + resources = self._render_instance.values.get("resources", {}) + gpus = resources.get("gpus", {}).get("nvidia_gpu_selection", {}) + if not gpus: + return + + for pci, gpu in gpus.items(): + if gpu.get("use_gpu", False): + if not gpu.get("uuid"): + raise RenderError(f"Expected [uuid] to be set for GPU in slot [{pci}] in [nvidia_gpu_selection]") + self._nvidia_ids.add(gpu["uuid"]) + + if self._nvidia_ids: + if not self._reservations: + self._reservations["devices"] = [] + self._reservations["devices"].append( + { + "capabilities": ["gpu"], + "driver": "nvidia", + "device_ids": sorted(self._nvidia_ids), + } + ) + + # This is only used on ix-app that we allow + # disabling cpus and memory. GPUs are only added + # if the user has requested them. + def remove_cpus_and_memory(self): + self._limits.pop("cpus", None) + self._limits.pop("memory", None) + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._reservations.pop("devices", None) + + def set_profile(self, profile: str): + cpu, memory = profile_mapping(profile) + self._set_cpu(cpu) + self._set_memory(memory) + + def has_resources(self): + return len(self._limits) > 0 or len(self._reservations) > 0 + + def has_gpus(self): + gpu_devices = [d for d in self._reservations.get("devices", []) if "gpu" in d["capabilities"]] + return len(gpu_devices) > 0 + + def render(self): + result = {} + if self._limits: + result["limits"] = self._limits + if self._reservations: + result["reservations"] = self._reservations + + return result + + +def profile_mapping(profile: str): + profiles = { + "low": (1, 512), + "medium": (2, 1024), + } + + if profile not in profiles: + raise RenderError( + f"Resource profile [{profile}] is not valid. Valid options are: [{', '.join(profiles.keys())}]" + ) + + return profiles[profile] diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/restart.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/restart.py new file mode 100644 index 0000000000..2f6281af48 --- /dev/null +++ b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/restart.py @@ -0,0 +1,25 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .validations import valid_restart_policy_or_raise +except ImportError: + from validations import valid_restart_policy_or_raise + + +class RestartPolicy: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._policy: str = "unless-stopped" + self._maximum_retry_count: int = 0 + + def set_policy(self, policy: str, maximum_retry_count: int = 0): + self._policy = valid_restart_policy_or_raise(policy, maximum_retry_count) + self._maximum_retry_count = maximum_retry_count + + def render(self): + if self._policy == "on-failure" and self._maximum_retry_count > 0: + return f"{self._policy}:{self._maximum_retry_count}" + return self._policy diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/storage.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/sysctls.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/__init__.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_build_image.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_build_image.py new file mode 100644 index 0000000000..f30c1210ed --- /dev/null +++ b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_build_image.py @@ -0,0 +1,49 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_build_image_with_from(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.build_image(["FROM test_image"]) + + +def test_build_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.build_image( + [ + "RUN echo hello", + None, + "", + "RUN echo world", + ] + ) + output = render.render() + assert ( + output["services"]["test_container"]["image"] + == "ix-nginx:latest_4a127145ea4c25511707e57005dd0ed457fe2f4932082c8f9faa339a450b6a99" + ) + assert output["services"]["test_container"]["build"] == { + "tags": ["ix-nginx:latest_4a127145ea4c25511707e57005dd0ed457fe2f4932082c8f9faa339a450b6a99"], + "dockerfile_inline": """FROM nginx:latest +RUN echo hello +RUN echo world +""", + } diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_configs.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_configs.py new file mode 100644 index 0000000000..9049e473ea --- /dev/null +++ b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_configs.py @@ -0,0 +1,63 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_duplicate_config_with_different_data(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.configs.add("test_config", "test_data", "/some/path") + with pytest.raises(Exception): + c1.configs.add("test_config", "test_data2", "/some/path") + + +def test_add_config_with_empty_target(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.configs.add("test_config", "test_data", "") + + +def test_add_duplicate_target(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.configs.add("test_config", "test_data", "/some/path") + with pytest.raises(Exception): + c1.configs.add("test_config2", "test_data2", "/some/path") + + +def test_add_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.configs.add("test_config", "$test_data", "/some/path") + output = render.render() + assert output["configs"]["test_config"]["content"] == "$$test_data" + assert output["services"]["test_container"]["configs"] == [{"source": "test_config", "target": "/some/path"}] + + +def test_add_config_with_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.configs.add("test_config", "test_data", "/some/path", "0777") + output = render.render() + assert output["configs"]["test_config"]["content"] == "test_data" + assert output["services"]["test_container"]["configs"] == [ + {"source": "test_config", "target": "/some/path", "mode": 511} + ] diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_container.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_container.py new file mode 100644 index 0000000000..61a22a5df2 --- /dev/null +++ b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_container.py @@ -0,0 +1,324 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_depends.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_depends.py new file mode 100644 index 0000000000..a1d8373927 --- /dev/null +++ b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_depends.py @@ -0,0 +1,54 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_dependency(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c2 = render.add_container("test_container2", "test_image") + c1.healthcheck.disable() + c2.healthcheck.disable() + c1.depends.add_dependency("test_container2", "service_started") + output = render.render() + assert output["services"]["test_container"]["depends_on"]["test_container2"] == {"condition": "service_started"} + + +def test_add_dependency_invalid_condition(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + render.add_container("test_container2", "test_image") + with pytest.raises(Exception): + c1.depends.add_dependency("test_container2", "invalid_condition") + + +def test_add_dependency_missing_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.depends.add_dependency("test_container2", "service_started") + + +def test_add_dependency_duplicate(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + render.add_container("test_container2", "test_image") + c1.depends.add_dependency("test_container2", "service_started") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.depends.add_dependency("test_container2", "service_started") diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_deps.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_deps.py new file mode 100644 index 0000000000..f9562ba4f2 --- /dev/null +++ b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_deps.py @@ -0,0 +1,380 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:latest" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_device.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_device.py new file mode 100644 index 0000000000..7455c829f6 --- /dev/null +++ b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_device.py @@ -0,0 +1,121 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_dns.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_dns.py new file mode 100644 index 0000000000..fe6b21e34f --- /dev/null +++ b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_dns.py @@ -0,0 +1,64 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_dns_opts(mock_values): + mock_values["network"] = {"dns_opts": ["attempts:3", "opt1", "opt2"]} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["dns_opt"] == ["attempts:3", "opt1", "opt2"] + + +def test_auto_add_dns_searches(mock_values): + mock_values["network"] = {"dns_searches": ["search1", "search2"]} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["dns_search"] == ["search1", "search2"] + + +def test_auto_add_dns_nameservers(mock_values): + mock_values["network"] = {"dns_nameservers": ["nameserver1", "nameserver2"]} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["dns"] == ["nameserver1", "nameserver2"] + + +def test_add_duplicate_dns_nameservers(mock_values): + mock_values["network"] = {"dns_nameservers": ["nameserver1", "nameserver1"]} + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_add_duplicate_dns_searches(mock_values): + mock_values["network"] = {"dns_searches": ["search1", "search1"]} + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_add_duplicate_dns_opts(mock_values): + mock_values["network"] = {"dns_opts": ["attempts:3", "attempts:5"]} + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_environment.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_environment.py new file mode 100644 index 0000000000..209f67551b --- /dev/null +++ b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_environment.py @@ -0,0 +1,184 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_formatter.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_formatter.py new file mode 100644 index 0000000000..843cf65d2e --- /dev/null +++ b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_formatter.py @@ -0,0 +1,13 @@ +from formatter import escape_dollar + + +def test_escape_dollar(): + cases = [ + {"input": "test", "expected": "test"}, + {"input": "$test", "expected": "$$test"}, + {"input": "$$test", "expected": "$$$$test"}, + {"input": "$$$test", "expected": "$$$$$$test"}, + {"input": "$test$", "expected": "$$test$$"}, + ] + for case in cases: + assert escape_dollar(case["input"]) == case["expected"] diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_functions.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_functions.py new file mode 100644 index 0000000000..13d5e49522 --- /dev/null +++ b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_functions.py @@ -0,0 +1,82 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_healthcheck.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_healthcheck.py new file mode 100644 index 0000000000..fbd488ece4 --- /dev/null +++ b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_healthcheck.py @@ -0,0 +1,187 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_labels.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_labels.py new file mode 100644 index 0000000000..ffa21eceac --- /dev/null +++ b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_labels.py @@ -0,0 +1,88 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_disallowed_label(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.labels.add_label("com.docker.compose.service", "test_service") + + +def test_add_duplicate_label(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.labels.add_label("my.custom.label", "test_value") + with pytest.raises(Exception): + c1.labels.add_label("my.custom.label", "test_value1") + + +def test_add_label_on_non_existing_container(mock_values): + mock_values["labels"] = [ + { + "key": "my.custom.label1", + "value": "test_value1", + "containers": ["test_container", "test_container2"], + }, + ] + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.render() + + +def test_add_label(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.labels.add_label("my.custom.label1", "test_value1") + c1.labels.add_label("my.custom.label2", "test_value2") + output = render.render() + assert output["services"]["test_container"]["labels"] == { + "my.custom.label1": "test_value1", + "my.custom.label2": "test_value2", + } + + +def test_auto_add_labels(mock_values): + mock_values["labels"] = [ + { + "key": "my.custom.label1", + "value": "test_value1", + "containers": ["test_container", "test_container2"], + }, + { + "key": "my.custom.label2", + "value": "test_value2", + "containers": ["test_container"], + }, + ] + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c2 = render.add_container("test_container2", "test_image") + c1.healthcheck.disable() + c2.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["labels"] == { + "my.custom.label1": "test_value1", + "my.custom.label2": "test_value2", + } + assert output["services"]["test_container2"]["labels"] == { + "my.custom.label1": "test_value1", + } diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_notes.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_notes.py new file mode 100644 index 0000000000..3613445385 --- /dev/null +++ b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_notes.py @@ -0,0 +1,213 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "ix_context": { + "app_metadata": { + "name": "test_app", + } + }, + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_notes(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert ( + output["x-notes"] + == """# Welcome to TrueNAS SCALE + +Thank you for installing test_app! + +## Documentation + +Documentation for test_app can be found at https://www.truenas.com/docs. + +## Bug reports + +If you find a bug in this app, please file an issue at +https://ixsystems.atlassian.net or https://github.com/truenas/apps + +## Feature requests or improvements + +If you find a feature request for this app, please file an issue at +https://ixsystems.atlassian.net or https://github.com/truenas/apps +""" + ) + + +def test_notes_with_warnings(mock_values): + render = Render(mock_values) + render.notes.add_warning("this is not properly configured. fix it now!") + render.notes.add_warning("that is not properly configured. fix it later!") + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert ( + output["x-notes"] + == """# Welcome to TrueNAS SCALE + +Thank you for installing test_app! + +## Warnings + +- this is not properly configured. fix it now! +- that is not properly configured. fix it later! + +## Documentation + +Documentation for test_app can be found at https://www.truenas.com/docs. + +## Bug reports + +If you find a bug in this app, please file an issue at +https://ixsystems.atlassian.net or https://github.com/truenas/apps + +## Feature requests or improvements + +If you find a feature request for this app, please file an issue at +https://ixsystems.atlassian.net or https://github.com/truenas/apps +""" + ) + + +def test_notes_with_deprecations(mock_values): + render = Render(mock_values) + render.notes.add_deprecation("this is will be removed later. fix it now!") + render.notes.add_deprecation("that is will be removed later. fix it later!") + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert ( + output["x-notes"] + == """# Welcome to TrueNAS SCALE + +Thank you for installing test_app! + +## Deprecations + +- this is will be removed later. fix it now! +- that is will be removed later. fix it later! + +## Documentation + +Documentation for test_app can be found at https://www.truenas.com/docs. + +## Bug reports + +If you find a bug in this app, please file an issue at +https://ixsystems.atlassian.net or https://github.com/truenas/apps + +## Feature requests or improvements + +If you find a feature request for this app, please file an issue at +https://ixsystems.atlassian.net or https://github.com/truenas/apps +""" + ) + + +def test_notes_with_body(mock_values): + render = Render(mock_values) + render.notes.set_body( + """## Additional info + +Some info +some other info. +""" + ) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert ( + output["x-notes"] + == """# Welcome to TrueNAS SCALE + +Thank you for installing test_app! + +## Additional info + +Some info +some other info. + +## Documentation + +Documentation for test_app can be found at https://www.truenas.com/docs. + +## Bug reports + +If you find a bug in this app, please file an issue at +https://ixsystems.atlassian.net or https://github.com/truenas/apps + +## Feature requests or improvements + +If you find a feature request for this app, please file an issue at +https://ixsystems.atlassian.net or https://github.com/truenas/apps +""" + ) + + +def test_notes_all(mock_values): + render = Render(mock_values) + render.notes.add_warning("this is not properly configured. fix it now!") + render.notes.add_warning("that is not properly configured. fix it later!") + render.notes.add_deprecation("this is will be removed later. fix it now!") + render.notes.add_deprecation("that is will be removed later. fix it later!") + render.notes.set_body( + """## Additional info + +Some info +some other info. +""" + ) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert ( + output["x-notes"] + == """# Welcome to TrueNAS SCALE + +Thank you for installing test_app! + +## Warnings + +- this is not properly configured. fix it now! +- that is not properly configured. fix it later! + +## Deprecations + +- this is will be removed later. fix it now! +- that is will be removed later. fix it later! + +## Additional info + +Some info +some other info. + +## Documentation + +Documentation for test_app can be found at https://www.truenas.com/docs. + +## Bug reports + +If you find a bug in this app, please file an issue at +https://ixsystems.atlassian.net or https://github.com/truenas/apps + +## Feature requests or improvements + +If you find a feature request for this app, please file an issue at +https://ixsystems.atlassian.net or https://github.com/truenas/apps +""" + ) diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_portal.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_portal.py new file mode 100644 index 0000000000..aebd9425c9 --- /dev/null +++ b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_portal.py @@ -0,0 +1,75 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_no_portals(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["x-portals"] == [] + + +def test_add_portal(mock_values): + render = Render(mock_values) + render.portals.add_portal({"scheme": "http", "path": "/", "port": 8080}) + render.portals.add_portal({"name": "Other Portal", "scheme": "https", "path": "/", "port": 8443}) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["x-portals"] == [ + {"name": "Other Portal", "scheme": "https", "host": "0.0.0.0", "port": 8443, "path": "/"}, + {"name": "Web UI", "scheme": "http", "host": "0.0.0.0", "port": 8080, "path": "/"}, + ] + + +def test_add_duplicate_portal(mock_values): + render = Render(mock_values) + render.portals.add_portal({"scheme": "http", "path": "/", "port": 8080}) + with pytest.raises(Exception): + render.portals.add_portal({"scheme": "http", "path": "/", "port": 8080}) + + +def test_add_duplicate_portal_with_explicit_name(mock_values): + render = Render(mock_values) + render.portals.add_portal({"name": "Some Portal", "scheme": "http", "path": "/", "port": 8080}) + with pytest.raises(Exception): + render.portals.add_portal({"name": "Some Portal", "scheme": "http", "path": "/", "port": 8080}) + + +def test_add_portal_with_invalid_scheme(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.portals.add_portal({"scheme": "invalid_scheme", "path": "/", "port": 8080}) + + +def test_add_portal_with_invalid_path(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.portals.add_portal({"scheme": "http", "path": "invalid_path", "port": 8080}) + + +def test_add_portal_with_invalid_path_double_slash(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.portals.add_portal({"scheme": "http", "path": "/some//path", "port": 8080}) + + +def test_add_portal_with_invalid_port(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.portals.add_portal({"scheme": "http", "path": "/", "port": -1}) diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_ports.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_ports.py new file mode 100644 index 0000000000..a4c923ca1d --- /dev/null +++ b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_ports.py @@ -0,0 +1,110 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_ports(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.ports.add_port(8081, 8080) + c1.ports.add_port(8082, 8080, {"protocol": "udp"}) + output = render.render() + assert output["services"]["test_container"]["ports"] == [ + {"published": 8081, "target": 8080, "protocol": "tcp", "mode": "ingress", "host_ip": "0.0.0.0"}, + {"published": 8082, "target": 8080, "protocol": "udp", "mode": "ingress", "host_ip": "0.0.0.0"}, + ] + + +def test_add_duplicate_ports(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.ports.add_port(8081, 8080) + c1.ports.add_port(8081, 8080, {"protocol": "udp"}) # This should not raise + with pytest.raises(Exception): + c1.ports.add_port(8081, 8080) + + +def test_add_duplicate_ports_with_different_host_ip(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.ports.add_port(8081, 8080, {"host_ip": "192.168.1.10"}) + c1.ports.add_port(8081, 8080, {"host_ip": "192.168.1.11"}) + output = render.render() + assert output["services"]["test_container"]["ports"] == [ + {"published": 8081, "target": 8080, "protocol": "tcp", "mode": "ingress", "host_ip": "192.168.1.10"}, + {"published": 8081, "target": 8080, "protocol": "tcp", "mode": "ingress", "host_ip": "192.168.1.11"}, + ] + + +def test_add_duplicate_ports_to_specific_host_ip_binds_to_0_0_0_0(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.ports.add_port(8081, 8080, {"host_ip": "192.168.1.10"}) + with pytest.raises(Exception): + c1.ports.add_port(8081, 8080, {"host_ip": "0.0.0.0"}) + + +def test_add_duplicate_ports_to_0_0_0_0_binds_to_specific_host_ip(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.ports.add_port(8081, 8080, {"host_ip": "0.0.0.0"}) + with pytest.raises(Exception): + c1.ports.add_port(8081, 8080, {"host_ip": "192.168.1.10"}) + + +def test_add_ports_with_invalid_protocol(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.ports.add_port(8081, 8080, {"protocol": "invalid_protocol"}) + + +def test_add_ports_with_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.ports.add_port(8081, 8080, {"mode": "invalid_mode"}) + + +def test_add_ports_with_invalid_ip(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.ports.add_port(8081, 8080, {"host_ip": "invalid_ip"}) + + +def test_add_ports_with_invalid_host_port(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.ports.add_port(-1, 8080) + + +def test_add_ports_with_invalid_container_port(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.ports.add_port(8081, -1) diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_render.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_render.py new file mode 100644 index 0000000000..60dc00679e --- /dev/null +++ b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_render.py @@ -0,0 +1,37 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_values_cannot_be_modified(mock_values): + render = Render(mock_values) + render.values["test"] = "test" + with pytest.raises(Exception): + render.render() + + +def test_duplicate_containers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_no_containers(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_resources.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_resources.py new file mode 100644 index 0000000000..cd83d164e5 --- /dev/null +++ b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_resources.py @@ -0,0 +1,140 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_automatically_add_cpu(mock_values): + mock_values["resources"] = {"limits": {"cpus": 1.0}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["deploy"]["resources"]["limits"]["cpus"] == "1.0" + + +def test_invalid_cpu(mock_values): + mock_values["resources"] = {"limits": {"cpus": "invalid"}} + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_automatically_add_memory(mock_values): + mock_values["resources"] = {"limits": {"memory": 1024}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["deploy"]["resources"]["limits"]["memory"] == "1024M" + + +def test_invalid_memory(mock_values): + mock_values["resources"] = {"limits": {"memory": "invalid"}} + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_automatically_add_gpus(mock_values): + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + devices = output["services"]["test_container"]["deploy"]["resources"]["reservations"]["devices"] + assert len(devices) == 1 + assert devices[0] == { + "capabilities": ["gpu"], + "driver": "nvidia", + "device_ids": ["uuid_0", "uuid_1"], + } + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_gpu_without_uuid(mock_values): + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_remove_cpus_and_memory_with_gpus(mock_values): + mock_values["resources"] = {"gpus": {"nvidia_gpu_selection": {"pci_slot_0": {"uuid": "uuid_1", "use_gpu": True}}}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.deploy.resources.remove_cpus_and_memory() + output = render.render() + assert "limits" not in output["services"]["test_container"]["deploy"]["resources"] + devices = output["services"]["test_container"]["deploy"]["resources"]["reservations"]["devices"] + assert len(devices) == 1 + assert devices[0] == { + "capabilities": ["gpu"], + "driver": "nvidia", + "device_ids": ["uuid_1"], + } + + +def test_remove_cpus_and_memory(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.deploy.resources.remove_cpus_and_memory() + output = render.render() + assert "deploy" not in output["services"]["test_container"] + + +def test_remove_devices(mock_values): + mock_values["resources"] = {"gpus": {"nvidia_gpu_selection": {"pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}}}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.deploy.resources.remove_devices() + output = render.render() + assert "reservations" not in output["services"]["test_container"]["deploy"]["resources"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_set_profile(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.deploy.resources.set_profile("low") + output = render.render() + assert output["services"]["test_container"]["deploy"]["resources"]["limits"]["cpus"] == "1" + assert output["services"]["test_container"]["deploy"]["resources"]["limits"]["memory"] == "512M" + + +def test_set_profile_invalid_profile(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.deploy.resources.set_profile("invalid_profile") diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_restart.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_restart.py new file mode 100644 index 0000000000..06b2975590 --- /dev/null +++ b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_restart.py @@ -0,0 +1,57 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_invalid_restart_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.restart.set_policy("invalid_policy") + + +def test_valid_restart_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.restart.set_policy("on-failure") + output = render.render() + assert output["services"]["test_container"]["restart"] == "on-failure" + + +def test_valid_restart_policy_with_maximum_retry_count(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.restart.set_policy("on-failure", 10) + output = render.render() + assert output["services"]["test_container"]["restart"] == "on-failure:10" + + +def test_invalid_restart_policy_with_maximum_retry_count(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.restart.set_policy("on-failure", maximum_retry_count=-1) + + +def test_invalid_restart_policy_with_maximum_retry_count_and_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.restart.set_policy("always", maximum_retry_count=10) diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_sysctls.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_volumes.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/validations.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/validations.py new file mode 100644 index 0000000000..13f155dfdb --- /dev/null +++ b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/validations.py @@ -0,0 +1,227 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/volume_mount.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/volume_mount.py new file mode 100644 index 0000000000..aadd077750 --- /dev/null +++ b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/volume_mount.py @@ -0,0 +1,92 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .formatter import merge_dicts_no_overwrite + from .volume_mount_types import BindMountType, VolumeMountType, TmpfsMountType + from .volume_sources import HostPathSource, IxVolumeSource, CifsSource, NfsSource, VolumeSource +except ImportError: + from error import RenderError + from formatter import merge_dicts_no_overwrite + from volume_mount_types import BindMountType, VolumeMountType, TmpfsMountType + from volume_sources import HostPathSource, IxVolumeSource, CifsSource, NfsSource, VolumeSource + + +class VolumeMount: + def __init__(self, render_instance: "Render", mount_path: str, config: "IxStorage"): + self._render_instance = render_instance + self.mount_path: str = mount_path + + storage_type: str = config.get("type", "") + if not storage_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match storage_type: + case "host_path": + spec_type = "bind" + mount_config = config.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + mount_type_specific_definition = BindMountType(self._render_instance, mount_config).render() + source = HostPathSource(self._render_instance, mount_config).get() + case "ix_volume": + spec_type = "bind" + mount_config = config.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + mount_type_specific_definition = BindMountType(self._render_instance, mount_config).render() + source = IxVolumeSource(self._render_instance, mount_config).get() + case "tmpfs": + spec_type = "tmpfs" + mount_config = config.get("tmpfs_config", {}) + mount_type_specific_definition = TmpfsMountType(self._render_instance, mount_config).render() + source = None + case "nfs": + spec_type = "volume" + mount_config = config.get("nfs_config") + if mount_config is None: + raise RenderError("Expected [nfs_config] to be set for [nfs] type.") + mount_type_specific_definition = VolumeMountType(self._render_instance, mount_config).render() + source = NfsSource(self._render_instance, mount_config).get() + case "cifs": + spec_type = "volume" + mount_config = config.get("cifs_config") + if mount_config is None: + raise RenderError("Expected [cifs_config] to be set for [cifs] type.") + mount_type_specific_definition = VolumeMountType(self._render_instance, mount_config).render() + source = CifsSource(self._render_instance, mount_config).get() + case "volume": + spec_type = "volume" + mount_config = config.get("volume_config") + if mount_config is None: + raise RenderError("Expected [volume_config] to be set for [volume] type.") + mount_type_specific_definition = VolumeMountType(self._render_instance, mount_config).render() + source = VolumeSource(self._render_instance, mount_config).get() + case "temporary": + spec_type = "volume" + mount_config = config.get("volume_config") + if mount_config is None: + raise RenderError("Expected [volume_config] to be set for [temporary] type.") + mount_type_specific_definition = VolumeMountType(self._render_instance, mount_config).render() + source = VolumeSource(self._render_instance, mount_config).get() + case "anonymous": + spec_type = "volume" + mount_config = config.get("volume_config") or {} + mount_type_specific_definition = VolumeMountType(self._render_instance, mount_config).render() + source = None + case _: + raise RenderError(f"Storage type [{storage_type}] is not supported for volume mounts.") + + common_spec = {"type": spec_type, "target": self.mount_path, "read_only": config.get("read_only", False)} + if source is not None: + common_spec["source"] = source + self._render_instance.volumes.add_volume(source, storage_type, mount_config) # type: ignore + + self.volume_mount_spec = merge_dicts_no_overwrite(common_spec, mount_type_specific_definition) + + def render(self) -> dict: + return self.volume_mount_spec diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/volume_mount_types.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/volume_mount_types.py new file mode 100644 index 0000000000..00a0ec3a18 --- /dev/null +++ b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/volume_mount_types.py @@ -0,0 +1,72 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorageTmpfsConfig, IxStorageVolumeConfig, IxStorageBindLikeConfigs + + +try: + from .error import RenderError + from .validations import valid_host_path_propagation, valid_octal_mode_or_raise +except ImportError: + from error import RenderError + from validations import valid_host_path_propagation, valid_octal_mode_or_raise + + +class TmpfsMountType: + def __init__(self, render_instance: "Render", config: "IxStorageTmpfsConfig"): + self._render_instance = render_instance + self.spec = {"tmpfs": {}} + size = config.get("size", None) + mode = config.get("mode", None) + + if size is not None: + if not isinstance(size, int): + raise RenderError(f"Expected [size] to be an integer for [tmpfs] type, got [{size}]") + if not size > 0: + raise RenderError(f"Expected [size] to be greater than 0 for [tmpfs] type, got [{size}]") + # Convert Mebibytes to Bytes + self.spec["tmpfs"]["size"] = size * 1024 * 1024 + + if mode is not None: + mode = valid_octal_mode_or_raise(mode) + self.spec["tmpfs"]["mode"] = int(mode, 8) + + if not self.spec["tmpfs"]: + self.spec.pop("tmpfs") + + def render(self) -> dict: + """Render the tmpfs mount specification.""" + return self.spec + + +class BindMountType: + def __init__(self, render_instance: "Render", config: "IxStorageBindLikeConfigs"): + self._render_instance = render_instance + self.spec: dict = {} + + propagation = valid_host_path_propagation(config.get("propagation", "rprivate")) + create_host_path = config.get("create_host_path", False) + + self.spec: dict = { + "bind": { + "create_host_path": create_host_path, + "propagation": propagation, + } + } + + def render(self) -> dict: + """Render the bind mount specification.""" + return self.spec + + +class VolumeMountType: + def __init__(self, render_instance: "Render", config: "IxStorageVolumeConfig"): + self._render_instance = render_instance + self.spec: dict = {} + + self.spec: dict = {"volume": {"nocopy": config.get("nocopy", False)}} + + def render(self) -> dict: + """Render the volume mount specification.""" + return self.spec diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/volume_sources.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/volume_sources.py new file mode 100644 index 0000000000..c33fe55ea1 --- /dev/null +++ b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/volume_sources.py @@ -0,0 +1,106 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorageHostPathConfig, IxStorageIxVolumeConfig, IxStorageVolumeConfig + +try: + from .error import RenderError + from .formatter import get_hashed_name_for_volume + from .validations import valid_fs_path_or_raise +except ImportError: + from error import RenderError + from formatter import get_hashed_name_for_volume + from validations import valid_fs_path_or_raise + + +class HostPathSource: + def __init__(self, render_instance: "Render", config: "IxStorageHostPathConfig"): + self._render_instance = render_instance + self.source: str = "" + + if not config: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + + path = "" + if config.get("acl_enable", False): + acl_path = config.get("acl", {}).get("path") + if not acl_path: + raise RenderError("Expected [host_path_config.acl.path] to be set for [host_path] type.") + path = valid_fs_path_or_raise(acl_path) + else: + path = valid_fs_path_or_raise(config.get("path", "")) + + self.source = path.rstrip("/") + + def get(self): + return self.source + + +class IxVolumeSource: + def __init__(self, render_instance: "Render", config: "IxStorageIxVolumeConfig"): + self._render_instance = render_instance + self.source: str = "" + + if not config: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + dataset_name = config.get("dataset_name") + if not dataset_name: + raise RenderError("Expected [ix_volume_config.dataset_name] to be set for [ix_volume] type.") + + ix_volumes = self._render_instance.values.get("ix_volumes", {}) + if dataset_name not in ix_volumes: + available = ", ".join(ix_volumes.keys()) + raise RenderError( + f"Expected the key [{dataset_name}] to be set in [ix_volumes] for [ix_volume] type. " + f"Available keys: [{available}]." + ) + + self.source = valid_fs_path_or_raise(ix_volumes[dataset_name].rstrip("/")) + + def get(self): + return self.source + + +class CifsSource: + def __init__(self, render_instance: "Render", config: dict): + self._render_instance = render_instance + self.source: str = "" + + if not config: + raise RenderError("Expected [cifs_config] to be set for [cifs] type.") + self.source = get_hashed_name_for_volume("cifs", config) + + def get(self): + return self.source + + +class NfsSource: + def __init__(self, render_instance: "Render", config: dict): + self._render_instance = render_instance + self.source: str = "" + + if not config: + raise RenderError("Expected [nfs_config] to be set for [nfs] type.") + self.source = get_hashed_name_for_volume("nfs", config) + + def get(self): + return self.source + + +class VolumeSource: + def __init__(self, render_instance: "Render", config: "IxStorageVolumeConfig"): + self._render_instance = render_instance + self.source: str = "" + + if not config: + raise RenderError("Expected [volume_config] to be set for [volume] type.") + + volume_name: str = config.get("volume_name", "") + if not volume_name: + raise RenderError("Expected [volume_config.volume_name] to be set for [volume] type.") + + self.source = volume_name + + def get(self): + return self.source diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/volume_types.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/volume_types.py new file mode 100644 index 0000000000..4ccea08f83 --- /dev/null +++ b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/volume_types.py @@ -0,0 +1,133 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorageNfsConfig, IxStorageCifsConfig, IxStorageVolumeConfig + + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_fs_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_fs_path_or_raise + + +class NfsVolume: + def __init__(self, render_instance: "Render", config: "IxStorageNfsConfig"): + self._render_instance = render_instance + + if not config: + raise RenderError("Expected [nfs_config] to be set for [nfs] type") + + required_keys = ["server", "path"] + for key in required_keys: + if not config.get(key): + raise RenderError(f"Expected [{key}] to be set for [nfs] type") + + opts = [f"addr={config['server']}"] + cfg_options = config.get("options") + if cfg_options: + if not isinstance(cfg_options, list): + raise RenderError("Expected [nfs_config.options] to be a list for [nfs] type") + + tracked_keys: set[str] = set() + disallowed_opts = ["addr"] + for opt in cfg_options: + if not isinstance(opt, str): + raise RenderError("Options for [nfs] type must be a list of strings.") + + key = opt.split("=")[0] + if key in tracked_keys: + raise RenderError(f"Option [{key}] already added for [nfs] type.") + if key in disallowed_opts: + raise RenderError(f"Option [{key}] is not allowed for [nfs] type.") + opts.append(opt) + tracked_keys.add(key) + + opts.sort() + + path = valid_fs_path_or_raise(config["path"].rstrip("/")) + self.volume_spec = { + "driver_opts": { + "type": "nfs", + "device": f":{path}", + "o": f"{','.join([escape_dollar(opt) for opt in opts])}", + }, + } + + def get(self): + return self.volume_spec + + +class CifsVolume: + def __init__(self, render_instance: "Render", config: "IxStorageCifsConfig"): + self._render_instance = render_instance + self.volume_spec: dict = {} + + if not config: + raise RenderError("Expected [cifs_config] to be set for [cifs] type") + + required_keys = ["server", "path", "username", "password"] + for key in required_keys: + if not config.get(key): + raise RenderError(f"Expected [{key}] to be set for [cifs] type") + + opts = [ + "noperm", + f"user={config['username']}", + f"password={config['password']}", + ] + + domain = config.get("domain") + if domain: + opts.append(f"domain={domain}") + + cfg_options = config.get("options") + if cfg_options: + if not isinstance(cfg_options, list): + raise RenderError("Expected [cifs_config.options] to be a list for [cifs] type") + + tracked_keys: set[str] = set() + disallowed_opts = ["user", "password", "domain", "noperm"] + for opt in cfg_options: + if not isinstance(opt, str): + raise RenderError("Options for [cifs] type must be a list of strings.") + + key = opt.split("=")[0] + if key in tracked_keys: + raise RenderError(f"Option [{key}] already added for [cifs] type.") + if key in disallowed_opts: + raise RenderError(f"Option [{key}] is not allowed for [cifs] type.") + for disallowed in disallowed_opts: + if key == disallowed: + raise RenderError(f"Option [{key}] is not allowed for [cifs] type.") + opts.append(opt) + tracked_keys.add(key) + opts.sort() + + server = config["server"].lstrip("/") + path = config["path"].strip("/") + path = valid_fs_path_or_raise("/" + path).lstrip("/") + + self.volume_spec = { + "driver_opts": { + "type": "cifs", + "device": f"//{server}/{path}", + "o": f"{','.join([escape_dollar(opt) for opt in opts])}", + }, + } + + def get(self): + return self.volume_spec + + +class DockerVolume: + def __init__(self, render_instance: "Render", config: "IxStorageVolumeConfig"): + self._render_instance = render_instance + self.volume_spec: dict = {} + + def get(self): + return self.volume_spec diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/volumes.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/volumes.py new file mode 100644 index 0000000000..e6925a402f --- /dev/null +++ b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/volumes.py @@ -0,0 +1,66 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + + +try: + from .error import RenderError + from .storage import IxStorageVolumeLikeConfigs + from .volume_types import NfsVolume, CifsVolume, DockerVolume +except ImportError: + from error import RenderError + from storage import IxStorageVolumeLikeConfigs + from volume_types import NfsVolume, CifsVolume, DockerVolume + + +class Volumes: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volumes: dict[str, Volume] = {} + + def add_volume( + self, + source: str, + storage_type: str, + config: "IxStorageVolumeLikeConfigs", + ): + # This method can be called many times from the volume mounts + # Only add the volume if it is not already added, but dont raise an error + if source == "": + raise RenderError(f"Volume source [{source}] cannot be empty") + + if source in self._volumes: + return + + self._volumes[source] = Volume(self._render_instance, storage_type, config) + + def has_volumes(self) -> bool: + return bool(self._volumes) + + def render(self): + return {name: v.render() for name, v in sorted(self._volumes.items()) if v.render() is not None} + + +class Volume: + def __init__( + self, + render_instance: "Render", + storage_type: str, + config: "IxStorageVolumeLikeConfigs", + ): + self._render_instance = render_instance + self.volume_spec: dict | None = {} + + match storage_type: + case "nfs": + self.volume_spec = NfsVolume(self._render_instance, config).get() # type: ignore + case "cifs": + self.volume_spec = CifsVolume(self._render_instance, config).get() # type: ignore + case "volume" | "temporary": + self.volume_spec = DockerVolume(self._render_instance, config).get() # type: ignore + case _: + self.volume_spec = None + + def render(self): + return self.volume_spec diff --git a/ix-dev/enterprise/syncthing/templates/macros/setup.sh.jinja b/ix-dev/enterprise/syncthing/templates/macros/setup.sh.jinja new file mode 100644 index 0000000000..2944e23bf8 --- /dev/null +++ b/ix-dev/enterprise/syncthing/templates/macros/setup.sh.jinja @@ -0,0 +1,45 @@ +{% macro setup_script(values, settings) -%} +{%- set config_dir = "%s/config"|format(values.consts.home_path) %} +{%- set cli = "syncthing cli --home %s config"|format(config_dir) %} +#!/bin/sh +set -e +trap cleanup EXIT TERM +cleanup() { + echo "Gracefully stopping Syncthing..." + if kill -0 $SYNCTHING_PID > /dev/null 2>&1; then + kill -SIGTERM $SYNCTHING_PID + wait $SYNCTHING_PID + fi + echo "Syncthing stopped." +} +try_for() { + local max_tries=$1 + local sleep_time=$2 + local cmd=$3 + tries=0 + until eval "$cmd"; do + [ $tries -ge $max_tries ] && return 1 + tries=$((tries+1)) + sleep $sleep_time + done +} +echo "Starting Syncthing in the background" +/bin/entrypoint.sh /bin/syncthing & +SYNCTHING_PID=$! +echo "Syncthing started with PID [$SYNCTHING_PID]" +echo "Waiting for Syncthing to be ready..." + +try_for 15 2 "[ -f '{{ config_dir }}/config.xml' ]" || { echo "Syncthing did not become ready in time. Exiting..."; exit 1; } +try_for 15 2 "curl --silent --output /dev/null http://127.0.0.1:{{ values.network.web_port }}/rest/noauth/health" || { echo "Syncthing did not become ready in time. Exiting..."; exit 1; } +echo "Syncthing is ready." + +{%- for cfg in settings %} +echo 'Using subcommand [{{ cfg.cmd }}] to set value [{{ '\"%s\"' | format(cfg.value) if cfg.quote else cfg.value }}]' +{{ cli }} {{ cfg.cmd }} set -- {{ '"%s"' | format(cfg.value) if cfg.quote else cfg.value }} || { echo "Failed to apply. Exiting..."; exit 1; } +{%- endfor %} + +echo "Gracefully stopping Syncthing..." +kill -SIGTERM $SYNCTHING_PID +wait $SYNCTHING_PID +echo "Syncthing stopped." +{%- endmacro %} diff --git a/ix-dev/enterprise/syncthing/templates/test_values/basic-values.yaml b/ix-dev/enterprise/syncthing/templates/test_values/basic-values.yaml index ef0ac1b566..64938b7ee3 100644 --- a/ix-dev/enterprise/syncthing/templates/test_values/basic-values.yaml +++ b/ix-dev/enterprise/syncthing/templates/test_values/basic-values.yaml @@ -18,26 +18,37 @@ run_as: user: 568 group: 568 +ix_volumes: + home: /opt/tests/mnt/home + data1: /opt/tests/mnt/data1 + data2: /opt/tests/mnt/data2 + storage: home: - type: volume - volume_name: syncthing-home - auto_permissions: true + type: ix_volume + ix_volume_config: + dataset_name: home + create_host_path: true additional_storage: - - type: volume + - type: ix_volume mount_path: /mnt/test/data1 + ix_volume_config: + dataset_name: data1 + create_host_path: true volume_name: test-data1 - - type: volume + - type: ix_volume mount_path: /mnt/test/data2 - volume_name: test-data2 + ix_volume_config: + dataset_name: data2 + create_host_path: true # Manual test for cifs rendering (migration_mode must add extra options) - # - type: cifs - # mount_path: /mnt/test/data3 - # volume_name: test-data3 - # cifs_config: - # server: 192.168.1.1 - # path: /test - # domain: WORKGROUP - # username: test - # password: test - # migration_mode: true + - type: cifs + mount_path: /mnt/test/data3 + volume_name: test-data3 + cifs_config: + server: 192.168.1.1 + path: /test + domain: WORKGROUP + username: test + password: test + migration_mode: true diff --git a/ix-dev/enterprise/syncthing/templates/test_values/https-values.yaml b/ix-dev/enterprise/syncthing/templates/test_values/https-values.yaml index 933dea991f..64ddf47ea1 100644 --- a/ix-dev/enterprise/syncthing/templates/test_values/https-values.yaml +++ b/ix-dev/enterprise/syncthing/templates/test_values/https-values.yaml @@ -18,18 +18,29 @@ run_as: user: 568 group: 568 +ix_volumes: + home: /opt/tests/mnt/home + data1: /opt/tests/mnt/data1 + data2: /opt/tests/mnt/data2 + storage: home: - type: volume - volume_name: syncthing-home - auto_permissions: true + type: ix_volume + ix_volume_config: + dataset_name: home + create_host_path: true additional_storage: - - type: volume + - type: ix_volume mount_path: /mnt/test/data1 + ix_volume_config: + dataset_name: data1 + create_host_path: true volume_name: test-data1 - - type: volume + - type: ix_volume mount_path: /mnt/test/data2 - volume_name: test-data2 + ix_volume_config: + dataset_name: data2 + create_host_path: true # Manual test for cifs rendering (migration_mode must add extra options) # - type: cifs # mount_path: /mnt/test/data3 From b3709e02e5fb1d7ac302bee18bdee8961b3037e6 Mon Sep 17 00:00:00 2001 From: Stavros Kois Date: Tue, 3 Dec 2024 18:02:57 +0200 Subject: [PATCH 10/19] update lib --- ix-dev/enterprise/syncthing/app.yaml | 4 +- .../syncthing/templates/docker-compose.yaml | 2 +- .../library/base_v1_1_7/environment.py | 98 ----- .../library/base_v1_1_7/healthchecks.py | 120 ------ .../templates/library/base_v1_1_7/mariadb.py | 72 ---- .../templates/library/base_v1_1_7/metadata.py | 71 ---- .../templates/library/base_v1_1_7/network.py | 21 - .../library/base_v1_1_7/permissions.py | 139 ------- .../templates/library/base_v1_1_7/ports.py | 42 -- .../templates/library/base_v1_1_7/postgres.py | 77 ---- .../templates/library/base_v1_1_7/redis.py | 49 --- .../library/base_v1_1_7/resources.py | 101 ----- .../templates/library/base_v1_1_7/security.py | 34 -- .../templates/library/base_v1_1_7/storage.py | 370 ------------------ .../templates/library/base_v1_1_7/utils.py | 124 ------ .../{base_v1_1_7 => base_v2_0_32}/__init__.py | 0 .../{base_v2_0_30 => base_v2_0_32}/configs.py | 0 .../container.py | 0 .../{base_v2_0_30 => base_v2_0_32}/depends.py | 0 .../{base_v2_0_30 => base_v2_0_32}/deploy.py | 0 .../{base_v2_0_30 => base_v2_0_32}/deps.py | 0 .../{base_v2_0_30 => base_v2_0_32}/device.py | 0 .../{base_v2_0_30 => base_v2_0_32}/devices.py | 0 .../{base_v2_0_30 => base_v2_0_32}/dns.py | 0 .../environment.py | 0 .../{base_v2_0_30 => base_v2_0_32}/error.py | 0 .../formatter.py | 0 .../library/base_v2_0_32}/functions.py | 3 +- .../healthcheck.py | 0 .../{base_v2_0_30 => base_v2_0_32}/labels.py | 0 .../{base_v2_0_30 => base_v2_0_32}/notes.py | 0 .../{base_v2_0_30 => base_v2_0_32}/portal.py | 0 .../{base_v2_0_30 => base_v2_0_32}/portals.py | 0 .../{base_v2_0_30 => base_v2_0_32}/ports.py | 0 .../{base_v2_0_30 => base_v2_0_32}/render.py | 0 .../resources.py | 0 .../{base_v2_0_30 => base_v2_0_32}/restart.py | 0 .../{base_v2_0_30 => base_v2_0_32}/storage.py | 0 .../{base_v2_0_30 => base_v2_0_32}/sysctls.py | 0 .../tests}/__init__.py | 0 .../tests/test_build_image.py | 0 .../tests/test_configs.py | 0 .../tests/test_container.py | 0 .../tests/test_depends.py | 0 .../tests/test_deps.py | 0 .../tests/test_device.py | 0 .../tests/test_dns.py | 0 .../tests/test_environment.py | 0 .../tests/test_formatter.py | 0 .../base_v2_0_32}/tests/test_functions.py | 0 .../tests/test_healthcheck.py | 0 .../tests/test_labels.py | 0 .../tests/test_notes.py | 0 .../tests/test_portal.py | 0 .../tests/test_ports.py | 0 .../tests/test_render.py | 0 .../tests/test_resources.py | 0 .../tests/test_restart.py | 0 .../tests/test_sysctls.py | 0 .../tests/test_volumes.py | 0 .../validations.py | 0 .../volume_mount.py | 0 .../volume_mount_types.py | 0 .../volume_sources.py | 0 .../volume_types.py | 0 .../{base_v2_0_30 => base_v2_0_32}/volumes.py | 0 .../templates/test_values/basic-values.yaml | 22 +- library/2.0.31/tests/__init__.py | 0 .../tests => library/2.0.32}/__init__.py | 0 library/{2.0.31 => 2.0.32}/configs.py | 0 library/{2.0.31 => 2.0.32}/container.py | 0 library/{2.0.31 => 2.0.32}/depends.py | 0 library/{2.0.31 => 2.0.32}/deploy.py | 0 library/{2.0.31 => 2.0.32}/deps.py | 0 library/{2.0.31 => 2.0.32}/device.py | 0 library/{2.0.31 => 2.0.32}/devices.py | 0 library/{2.0.31 => 2.0.32}/dns.py | 0 library/{2.0.31 => 2.0.32}/environment.py | 0 library/{2.0.31 => 2.0.32}/error.py | 0 library/{2.0.31 => 2.0.32}/formatter.py | 0 .../2.0.32}/functions.py | 9 +- library/{2.0.31 => 2.0.32}/healthcheck.py | 0 library/{2.0.31 => 2.0.32}/labels.py | 0 library/{2.0.31 => 2.0.32}/notes.py | 0 library/{2.0.31 => 2.0.32}/portal.py | 0 library/{2.0.31 => 2.0.32}/portals.py | 0 library/{2.0.31 => 2.0.32}/ports.py | 0 library/{2.0.31 => 2.0.32}/render.py | 0 library/{2.0.31 => 2.0.32}/resources.py | 0 library/{2.0.31 => 2.0.32}/restart.py | 0 library/{2.0.31 => 2.0.32}/storage.py | 0 library/{2.0.31 => 2.0.32}/sysctls.py | 0 library/{2.0.31 => 2.0.32/tests}/__init__.py | 0 .../tests/test_build_image.py | 0 .../{2.0.31 => 2.0.32}/tests/test_configs.py | 0 .../tests/test_container.py | 0 .../{2.0.31 => 2.0.32}/tests/test_depends.py | 0 library/{2.0.31 => 2.0.32}/tests/test_deps.py | 0 .../{2.0.31 => 2.0.32}/tests/test_device.py | 0 library/{2.0.31 => 2.0.32}/tests/test_dns.py | 0 .../tests/test_environment.py | 0 .../tests/test_formatter.py | 0 .../2.0.32}/tests/test_functions.py | 6 + .../tests/test_healthcheck.py | 0 .../{2.0.31 => 2.0.32}/tests/test_labels.py | 0 .../{2.0.31 => 2.0.32}/tests/test_notes.py | 0 .../{2.0.31 => 2.0.32}/tests/test_portal.py | 0 .../{2.0.31 => 2.0.32}/tests/test_ports.py | 0 .../{2.0.31 => 2.0.32}/tests/test_render.py | 0 .../tests/test_resources.py | 0 .../{2.0.31 => 2.0.32}/tests/test_restart.py | 0 .../{2.0.31 => 2.0.32}/tests/test_sysctls.py | 0 .../{2.0.31 => 2.0.32}/tests/test_volumes.py | 0 library/{2.0.31 => 2.0.32}/validations.py | 0 library/{2.0.31 => 2.0.32}/volume_mount.py | 0 .../{2.0.31 => 2.0.32}/volume_mount_types.py | 0 library/{2.0.31 => 2.0.32}/volume_sources.py | 0 library/{2.0.31 => 2.0.32}/volume_types.py | 0 library/{2.0.31 => 2.0.32}/volumes.py | 0 library/hashes.yaml | 2 +- 120 files changed, 31 insertions(+), 1335 deletions(-) delete mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/environment.py delete mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/healthchecks.py delete mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/mariadb.py delete mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/metadata.py delete mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/network.py delete mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/permissions.py delete mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/ports.py delete mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/postgres.py delete mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/redis.py delete mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/resources.py delete mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/security.py delete mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/storage.py delete mode 100644 ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/utils.py rename ix-dev/enterprise/syncthing/templates/library/{base_v1_1_7 => base_v2_0_32}/__init__.py (100%) rename ix-dev/enterprise/syncthing/templates/library/{base_v2_0_30 => base_v2_0_32}/configs.py (100%) rename ix-dev/enterprise/syncthing/templates/library/{base_v2_0_30 => base_v2_0_32}/container.py (100%) rename ix-dev/enterprise/syncthing/templates/library/{base_v2_0_30 => base_v2_0_32}/depends.py (100%) rename ix-dev/enterprise/syncthing/templates/library/{base_v2_0_30 => base_v2_0_32}/deploy.py (100%) rename ix-dev/enterprise/syncthing/templates/library/{base_v2_0_30 => base_v2_0_32}/deps.py (100%) rename ix-dev/enterprise/syncthing/templates/library/{base_v2_0_30 => base_v2_0_32}/device.py (100%) rename ix-dev/enterprise/syncthing/templates/library/{base_v2_0_30 => base_v2_0_32}/devices.py (100%) rename ix-dev/enterprise/syncthing/templates/library/{base_v2_0_30 => base_v2_0_32}/dns.py (100%) rename ix-dev/enterprise/syncthing/templates/library/{base_v2_0_30 => base_v2_0_32}/environment.py (100%) rename ix-dev/enterprise/syncthing/templates/library/{base_v2_0_30 => base_v2_0_32}/error.py (100%) rename ix-dev/enterprise/syncthing/templates/library/{base_v2_0_30 => base_v2_0_32}/formatter.py (100%) rename {library/2.0.31 => ix-dev/enterprise/syncthing/templates/library/base_v2_0_32}/functions.py (99%) rename ix-dev/enterprise/syncthing/templates/library/{base_v2_0_30 => base_v2_0_32}/healthcheck.py (100%) rename ix-dev/enterprise/syncthing/templates/library/{base_v2_0_30 => base_v2_0_32}/labels.py (100%) rename ix-dev/enterprise/syncthing/templates/library/{base_v2_0_30 => base_v2_0_32}/notes.py (100%) rename ix-dev/enterprise/syncthing/templates/library/{base_v2_0_30 => base_v2_0_32}/portal.py (100%) rename ix-dev/enterprise/syncthing/templates/library/{base_v2_0_30 => base_v2_0_32}/portals.py (100%) rename ix-dev/enterprise/syncthing/templates/library/{base_v2_0_30 => base_v2_0_32}/ports.py (100%) rename ix-dev/enterprise/syncthing/templates/library/{base_v2_0_30 => base_v2_0_32}/render.py (100%) rename ix-dev/enterprise/syncthing/templates/library/{base_v2_0_30 => base_v2_0_32}/resources.py (100%) rename ix-dev/enterprise/syncthing/templates/library/{base_v2_0_30 => base_v2_0_32}/restart.py (100%) rename ix-dev/enterprise/syncthing/templates/library/{base_v2_0_30 => base_v2_0_32}/storage.py (100%) rename ix-dev/enterprise/syncthing/templates/library/{base_v2_0_30 => base_v2_0_32}/sysctls.py (100%) rename ix-dev/enterprise/syncthing/templates/library/{base_v2_0_30 => base_v2_0_32/tests}/__init__.py (100%) rename ix-dev/enterprise/syncthing/templates/library/{base_v2_0_30 => base_v2_0_32}/tests/test_build_image.py (100%) rename ix-dev/enterprise/syncthing/templates/library/{base_v2_0_30 => base_v2_0_32}/tests/test_configs.py (100%) rename ix-dev/enterprise/syncthing/templates/library/{base_v2_0_30 => base_v2_0_32}/tests/test_container.py (100%) rename ix-dev/enterprise/syncthing/templates/library/{base_v2_0_30 => base_v2_0_32}/tests/test_depends.py (100%) rename ix-dev/enterprise/syncthing/templates/library/{base_v2_0_30 => base_v2_0_32}/tests/test_deps.py (100%) rename ix-dev/enterprise/syncthing/templates/library/{base_v2_0_30 => base_v2_0_32}/tests/test_device.py (100%) rename ix-dev/enterprise/syncthing/templates/library/{base_v2_0_30 => base_v2_0_32}/tests/test_dns.py (100%) rename ix-dev/enterprise/syncthing/templates/library/{base_v2_0_30 => base_v2_0_32}/tests/test_environment.py (100%) rename ix-dev/enterprise/syncthing/templates/library/{base_v2_0_30 => base_v2_0_32}/tests/test_formatter.py (100%) rename {library/2.0.31 => ix-dev/enterprise/syncthing/templates/library/base_v2_0_32}/tests/test_functions.py (100%) rename ix-dev/enterprise/syncthing/templates/library/{base_v2_0_30 => base_v2_0_32}/tests/test_healthcheck.py (100%) rename ix-dev/enterprise/syncthing/templates/library/{base_v2_0_30 => base_v2_0_32}/tests/test_labels.py (100%) rename ix-dev/enterprise/syncthing/templates/library/{base_v2_0_30 => base_v2_0_32}/tests/test_notes.py (100%) rename ix-dev/enterprise/syncthing/templates/library/{base_v2_0_30 => base_v2_0_32}/tests/test_portal.py (100%) rename ix-dev/enterprise/syncthing/templates/library/{base_v2_0_30 => base_v2_0_32}/tests/test_ports.py (100%) rename ix-dev/enterprise/syncthing/templates/library/{base_v2_0_30 => base_v2_0_32}/tests/test_render.py (100%) rename ix-dev/enterprise/syncthing/templates/library/{base_v2_0_30 => base_v2_0_32}/tests/test_resources.py (100%) rename ix-dev/enterprise/syncthing/templates/library/{base_v2_0_30 => base_v2_0_32}/tests/test_restart.py (100%) rename ix-dev/enterprise/syncthing/templates/library/{base_v2_0_30 => base_v2_0_32}/tests/test_sysctls.py (100%) rename ix-dev/enterprise/syncthing/templates/library/{base_v2_0_30 => base_v2_0_32}/tests/test_volumes.py (100%) rename ix-dev/enterprise/syncthing/templates/library/{base_v2_0_30 => base_v2_0_32}/validations.py (100%) rename ix-dev/enterprise/syncthing/templates/library/{base_v2_0_30 => base_v2_0_32}/volume_mount.py (100%) rename ix-dev/enterprise/syncthing/templates/library/{base_v2_0_30 => base_v2_0_32}/volume_mount_types.py (100%) rename ix-dev/enterprise/syncthing/templates/library/{base_v2_0_30 => base_v2_0_32}/volume_sources.py (100%) rename ix-dev/enterprise/syncthing/templates/library/{base_v2_0_30 => base_v2_0_32}/volume_types.py (100%) rename ix-dev/enterprise/syncthing/templates/library/{base_v2_0_30 => base_v2_0_32}/volumes.py (100%) delete mode 100644 library/2.0.31/tests/__init__.py rename {ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests => library/2.0.32}/__init__.py (100%) rename library/{2.0.31 => 2.0.32}/configs.py (100%) rename library/{2.0.31 => 2.0.32}/container.py (100%) rename library/{2.0.31 => 2.0.32}/depends.py (100%) rename library/{2.0.31 => 2.0.32}/deploy.py (100%) rename library/{2.0.31 => 2.0.32}/deps.py (100%) rename library/{2.0.31 => 2.0.32}/device.py (100%) rename library/{2.0.31 => 2.0.32}/devices.py (100%) rename library/{2.0.31 => 2.0.32}/dns.py (100%) rename library/{2.0.31 => 2.0.32}/environment.py (100%) rename library/{2.0.31 => 2.0.32}/error.py (100%) rename library/{2.0.31 => 2.0.32}/formatter.py (100%) rename {ix-dev/enterprise/syncthing/templates/library/base_v2_0_30 => library/2.0.32}/functions.py (93%) rename library/{2.0.31 => 2.0.32}/healthcheck.py (100%) rename library/{2.0.31 => 2.0.32}/labels.py (100%) rename library/{2.0.31 => 2.0.32}/notes.py (100%) rename library/{2.0.31 => 2.0.32}/portal.py (100%) rename library/{2.0.31 => 2.0.32}/portals.py (100%) rename library/{2.0.31 => 2.0.32}/ports.py (100%) rename library/{2.0.31 => 2.0.32}/render.py (100%) rename library/{2.0.31 => 2.0.32}/resources.py (100%) rename library/{2.0.31 => 2.0.32}/restart.py (100%) rename library/{2.0.31 => 2.0.32}/storage.py (100%) rename library/{2.0.31 => 2.0.32}/sysctls.py (100%) rename library/{2.0.31 => 2.0.32/tests}/__init__.py (100%) rename library/{2.0.31 => 2.0.32}/tests/test_build_image.py (100%) rename library/{2.0.31 => 2.0.32}/tests/test_configs.py (100%) rename library/{2.0.31 => 2.0.32}/tests/test_container.py (100%) rename library/{2.0.31 => 2.0.32}/tests/test_depends.py (100%) rename library/{2.0.31 => 2.0.32}/tests/test_deps.py (100%) rename library/{2.0.31 => 2.0.32}/tests/test_device.py (100%) rename library/{2.0.31 => 2.0.32}/tests/test_dns.py (100%) rename library/{2.0.31 => 2.0.32}/tests/test_environment.py (100%) rename library/{2.0.31 => 2.0.32}/tests/test_formatter.py (100%) rename {ix-dev/enterprise/syncthing/templates/library/base_v2_0_30 => library/2.0.32}/tests/test_functions.py (93%) rename library/{2.0.31 => 2.0.32}/tests/test_healthcheck.py (100%) rename library/{2.0.31 => 2.0.32}/tests/test_labels.py (100%) rename library/{2.0.31 => 2.0.32}/tests/test_notes.py (100%) rename library/{2.0.31 => 2.0.32}/tests/test_portal.py (100%) rename library/{2.0.31 => 2.0.32}/tests/test_ports.py (100%) rename library/{2.0.31 => 2.0.32}/tests/test_render.py (100%) rename library/{2.0.31 => 2.0.32}/tests/test_resources.py (100%) rename library/{2.0.31 => 2.0.32}/tests/test_restart.py (100%) rename library/{2.0.31 => 2.0.32}/tests/test_sysctls.py (100%) rename library/{2.0.31 => 2.0.32}/tests/test_volumes.py (100%) rename library/{2.0.31 => 2.0.32}/validations.py (100%) rename library/{2.0.31 => 2.0.32}/volume_mount.py (100%) rename library/{2.0.31 => 2.0.32}/volume_mount_types.py (100%) rename library/{2.0.31 => 2.0.32}/volume_sources.py (100%) rename library/{2.0.31 => 2.0.32}/volume_types.py (100%) rename library/{2.0.31 => 2.0.32}/volumes.py (100%) diff --git a/ix-dev/enterprise/syncthing/app.yaml b/ix-dev/enterprise/syncthing/app.yaml index d6cc6ba0af..af06f4c570 100644 --- a/ix-dev/enterprise/syncthing/app.yaml +++ b/ix-dev/enterprise/syncthing/app.yaml @@ -25,8 +25,8 @@ icon: https://media.sys.truenas.net/apps/syncthing/icons/icon.svg keywords: - sync - file-sharing -lib_version: 2.0.30 -lib_version_hash: a87750c752394f7bf0eb461678564a9ef99bc4e4541787ef25e7be1095a6f879 +lib_version: 2.0.32 +lib_version_hash: 4a0bf69cccda322e191eab36ab81ca6d0c8e5d64a0b2fa117c609804b55b86c6 maintainers: - email: dev@ixsystems.com name: truenas diff --git a/ix-dev/enterprise/syncthing/templates/docker-compose.yaml b/ix-dev/enterprise/syncthing/templates/docker-compose.yaml index 871dec871f..9e813501da 100644 --- a/ix-dev/enterprise/syncthing/templates/docker-compose.yaml +++ b/ix-dev/enterprise/syncthing/templates/docker-compose.yaml @@ -63,7 +63,7 @@ {% if values.network.certificate_id %} {% set cert = values.ix_certificates[values.network.certificate_id] %} - {% do c1.configs.add("private", cert.certificate, "%s/config/https-key.pem" | format(values.consts.home_path)) %} + {% do c1.configs.add("private", cert.privatekey, "%s/config/https-key.pem" | format(values.consts.home_path)) %} {% do c1.configs.add("public", cert.certificate, "%s/config/https-cert.pem" | format(values.consts.home_path)) %} {% do config.configs.add("private", cert.privatekey, "%s/config/https-key.pem" | format(values.consts.home_path)) %} diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/environment.py b/ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/environment.py deleted file mode 100644 index be5c8b7347..0000000000 --- a/ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/environment.py +++ /dev/null @@ -1,98 +0,0 @@ -from . import utils -from .resources import get_nvidia_gpus_reservations - - -def envs(app: dict | None = None, user: list | None = None, values: dict | None = None): - app = app or {} - user = user or [] - values = values or {} - result = {} - - if not values: - utils.throw_error("Values cannot be empty in environment.py") - - if not isinstance(user, list): - utils.throw_error( - f"Unsupported type for user environment variables [{type(user)}]" - ) - - # Always set TZ - result.update({"TZ": values.get("TZ", "Etc/UTC")}) - - # Update envs with nvidia variables - if values.get("resources", {}).get("gpus", {}): - result.update(get_nvidia_env(values.get("resources", {}).get("gpus", {}))) - - # Update envs with run_as variables - if values.get("run_as"): - result.update(get_run_as_envs(values.get("run_as", {}))) - - # Make sure we don't manually set any of the above - for item in app.items(): - if not item[0]: - utils.throw_error("Environment variable name cannot be empty.") - if item[0] in result: - utils.throw_error( - f"Environment variable [{item[0]}] is already defined automatically from the library." - ) - result[item[0]] = item[1] - - for item in user: - if not item.get("name"): - utils.throw_error("Environment variable name cannot be empty.") - if item.get("name") in result: - utils.throw_error( - f"Environment variable [{item['name']}] is already defined from the application developer." - ) - result[item["name"]] = item.get("value") - - for k, v in result.items(): - val = str(v) - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - val = val.lower() - result[k] = utils.escape_dollar(val) - - return result - - -# Sets some common variables that most applications use -def get_run_as_envs(run_as: dict) -> dict: - result = {} - user = run_as.get("user") - group = run_as.get("group") - if user: - result.update( - { - "PUID": user, - "UID": user, - "USER_ID": user, - } - ) - if group: - result.update( - { - "PGID": group, - "GID": group, - "GROUP_ID": group, - } - ) - return result - - -def get_nvidia_env(gpus: dict) -> dict: - reservations = get_nvidia_gpus_reservations(gpus) - if not reservations.get("device_ids"): - return { - "NVIDIA_VISIBLE_DEVICES": "void", - } - - return { - "NVIDIA_VISIBLE_DEVICES": ( - ",".join(reservations["device_ids"]) - if reservations.get("device_ids") - else "void" - ), - "NVIDIA_DRIVER_CAPABILITIES": "all", - } diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/healthchecks.py b/ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/healthchecks.py deleted file mode 100644 index cc98270d1d..0000000000 --- a/ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/healthchecks.py +++ /dev/null @@ -1,120 +0,0 @@ -from . import utils - - -def check_health(test, interval=10, timeout=5, retries=30, start_period=10): - if not test: - utils.throw_error("Expected [test] to be set") - - return { - "test": test, - "interval": f"{interval}s", - "timeout": f"{timeout}s", - "retries": retries, - "start_period": f"{start_period}s", - } - - -def mariadb_test(db, config=None): - config = config or {} - if not db: - utils.throw_error("MariaDB container: [db] must be set") - - host = config.get("host", "127.0.0.1") - port = config.get("port", 3306) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$$MARIADB_ROOT_PASSWORD ping" - - -def pg_test(user, db, config=None): - config = config or {} - if not user or not db: - utils.throw_error("Postgres container: [user] and [db] must be set") - - host = config.get("host", "127.0.0.1") - port = config.get("port", 5432) - - return f"pg_isready -h {host} -p {port} -d {db} -U {user}" - - -def redis_test(config=None): - config = config or {} - - host = config.get("host", "127.0.0.1") - port = config.get("port", 6379) - password = "$$REDIS_PASSWORD" - - return f"redis-cli -h {host} -p {port} -a {password} ping | grep -q PONG" - - -def curl_test(port, path, config=None): - config = config or {} - if not port or not path: - utils.throw_error("Expected [port] and [path] to be set") - - scheme = config.get("scheme", "http") - host = config.get("host", "127.0.0.1") - headers = config.get("headers", []) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - utils.throw_error("Expected [header] to be a list of two items") - opts.append(f'--header "{header[0]}: {header[1]}"') - - return f"curl --silent --output /dev/null --show-error --fail {' '.join(opts)} {scheme}://{host}:{port}{path}" - - -def wget_test(port, path, config=None): - config = config or {} - if not port or not path: - utils.throw_error("Expected [port] and [path] to be set") - - scheme = config.get("scheme", "http") - host = config.get("host", "127.0.0.1") - headers = config.get("headers", []) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - utils.throw_error("Expected [header] to be a list of two items") - opts.append(f'--header "{header[0]}: {header[1]}"') - - return f"wget --spider --quiet {' '.join(opts)} {scheme}://{host}:{port}{path}" - - -def http_test(port, path, config=None): - config = config or {} - if not port or not path: - utils.throw_error("Expected [port] and [path] to be set") - - host = config.get("host", "127.0.0.1") - - return f""" - /bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&$${{hc_fd}} && cat <&$${{hc_fd}} | grep "HTTP/1.1 200"' - """ # noqa - - -def netcat_test(port, config=None): - config = config or {} - if not port: - utils.throw_error("Expected [port] to be set") - - host = config.get("host", "127.0.0.1") - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(port, config=None): - config = config or {} - if not port: - utils.throw_error("Expected [port] to be set") - - host = config.get("host", "127.0.0.1") - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/mariadb.py b/ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/mariadb.py deleted file mode 100644 index fc12daf81f..0000000000 --- a/ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/mariadb.py +++ /dev/null @@ -1,72 +0,0 @@ -from . import utils -from .security import get_caps, get_sec_opts -from .network import dns_opts -from .healthchecks import mariadb_test, check_health -from .resources import resources - - -def mariadb_env(user, password, root_password, dbname): - if not user: - utils.throw_error("Expected [user] to be set for mariadb") - if not password: - utils.throw_error("Expected [password] to be set for mariadb") - if not root_password: - utils.throw_error("Expected [root_password] to be set for mariadb") - if not dbname: - utils.throw_error("Expected [dbname] to be set for mariadb") - return { - "MARIADB_USER": user, - "MARIADB_PASSWORD": utils.escape_dollar(password), - "MARIADB_ROOT_PASSWORD": utils.escape_dollar(root_password), - "MARIADB_DATABASE": dbname, - "MARIADB_AUTO_UPGRADE": "true", - } - - -def mariadb_container(data={}): - req_keys = [ - "db_user", - "db_password", - "db_root_password", - "db_name", - "volumes", - "resources", - ] - for key in req_keys: - if not data.get(key): - utils.throw_error(f"Expected [{key}] to be set for mariadb") - - db_user = data["db_user"] - db_password = data["db_password"] - db_root_password = data["db_root_password"] - db_name = data["db_name"] - db_port = data.get("port", 3306) - depends = data.get("depends_on", {}) - depends_on = {} - for key in depends: - depends_on[key] = { - "condition": depends[key].get("condition", "service_completed_successfully") - } - - return { - "image": f"{data.get('image', 'mariadb:10.6')}", - "user": f"{data.get('user', '999')}:{data.get('group', '999')}", - "restart": "unless-stopped", - "cap_drop": get_caps()["drop"], - "security_opt": get_sec_opts(), - **({"dns_opts": dns_opts(data["dns_opts"])} if data.get("dns_opts") else {}), - "healthcheck": check_health(mariadb_test(db=db_name, config={"port": db_port})), - "command": [ - "--port", - str(db_port), - ], - "environment": mariadb_env( - user=db_user, - password=db_password, - root_password=db_root_password, - dbname=db_name, - ), - "volumes": data["volumes"], - "depends_on": depends_on, - "deploy": {"resources": resources(data["resources"])}, - } diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/metadata.py b/ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/metadata.py deleted file mode 100644 index c0a59f8979..0000000000 --- a/ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/metadata.py +++ /dev/null @@ -1,71 +0,0 @@ -from . import utils - - -def get_header(app_name: str): - return f"""# Welcome to TrueNAS SCALE - -Thank you for installing {app_name}! -""" - - -def get_footer(app_name: str): - return f"""## Documentation - -Documentation for {app_name} can be found at https://www.truenas.com/docs. - -## Bug reports - -If you find a bug in this app, please file an issue at -https://ixsystems.atlassian.net or https://github.com/truenas/apps - -## Feature requests or improvements - -If you find a feature request for this app, please file an issue at -https://ixsystems.atlassian.net or https://github.com/truenas/apps -""" - - -def get_notes(app_name: str, body: str = ""): - if not app_name: - utils.throw_error("Expected [app_name] to be set") - - return f"{get_header(app_name)}\n\n{body}\n\n{get_footer(app_name)}" - - -def get_portals(portals: list): - valid_schemes = ["http", "https"] - result = [] - for portal in portals: - # Most apps have a single portal, lets default to a standard name - name = portal.get("name", "Web UI") - scheme = portal.get("scheme", "http") - path = portal.get("path", "/") - - if not name: - utils.throw_error("Expected [portal.name] to be set") - if name in [p["name"] for p in result]: - utils.throw_error( - f"Expected [portal.name] to be unique, got [{', '.join([p['name'] for p in result]+[name])}]" - ) - if scheme not in valid_schemes: - utils.throw_error( - f"Expected [portal.scheme] to be one of [{', '.join(valid_schemes)}], got [{portal['scheme']}]" - ) - if not portal.get("port"): - utils.throw_error("Expected [portal.port] to be set") - if not path.startswith("/"): - utils.throw_error( - f"Expected [portal.path] to start with /, got [{portal['path']}]" - ) - - result.append( - { - "name": name, - "scheme": scheme, - "host": portal.get("host", "0.0.0.0"), - "port": portal["port"], - "path": path, - } - ) - - return result diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/network.py b/ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/network.py deleted file mode 100644 index e4761fd295..0000000000 --- a/ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/network.py +++ /dev/null @@ -1,21 +0,0 @@ -from . import utils - - -def dns_opts(dns_options=None): - dns_options = dns_options or [] - if not dns_options: - return [] - - tracked = {} - disallowed_opts = [] - for opt in dns_options: - key = opt.split(":")[0] - if key in tracked: - utils.throw_error( - f"Expected [dns_opts] to be unique, got [{', '.join([d.split(':')[0] for d in tracked])}]" - ) - if key in disallowed_opts: - utils.throw_error(f"Expected [dns_opts] to not contain [{key}] key.") - tracked[key] = opt - - return dns_options diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/permissions.py b/ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/permissions.py deleted file mode 100644 index 1ce6e60ca1..0000000000 --- a/ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/permissions.py +++ /dev/null @@ -1,139 +0,0 @@ -import jsonschema - -from . import utils - -ITEM_SCHEMA = { - "type": "object", - "properties": { - "dir": {"type": "string"}, - "mode": {"type": "string", "enum": ["always", "check"]}, - "uid": {"type": "integer"}, - "gid": {"type": "integer"}, - "chmod": {"type": "string"}, - "is_temporary": {"type": "boolean"}, - }, - "required": ["dir", "mode", "uid", "gid", "chmod", "is_temporary"], -} - - -def perms_container(items=[], volumes=[]): - if not items: - raise ValueError("Expected [items] to be set for perms_container") - if not volumes: - raise ValueError("Expected [volumes] to be set for perms_container") - - command = [process_dir_shell_func()] - for item in items: - try: - jsonschema.validate(item, ITEM_SCHEMA) - except jsonschema.ValidationError as e: - utils.throw_error(f"Item [{item}] is not valid: {e}") - cmd = [ - "process_dir", - item["dir"], - item["mode"], - str(item["uid"]), - str(item["gid"]), - item["chmod"], - str(item["is_temporary"]).lower(), - ] - command.append(" ".join(cmd)) - - return { - "image": "bash", - "user": "root", - "deploy": { - "resources": { - "limits": {"cpus": "1.0", "memory": "512m"}, - } - }, - "entrypoint": ["bash", "-c"], - "command": ["\n".join(command)], - "volumes": volumes, - } - - -# Don't forget to use double $ for shell variables, -# otherwise docker-compose will try to expand them -def process_dir_shell_func(): - return """ -function process_dir() { - local dir=$$1 - local mode=$$2 - local uid=$$3 - local gid=$$4 - local chmod=$$5 - local is_temporary=$$6 - - local fix_owner="false" - local fix_perms="false" - - if [ -z "$$dir" ]; then - echo "Path is empty, skipping..." - return 0 - fi - - if [ ! -d "$$dir" ]; then - echo "Path [$$dir] does is not a directory, skipping..." - return 0 - fi - - if [ "$$is_temporary" = "true" ]; then - echo "Path [$$dir] is a temporary directory, ensuring it is empty..." - # Exclude the safe directory, where we can use to mount files temporarily - find "$$dir" -mindepth 1 -maxdepth 1 ! -name "ix-safe" -exec rm -rf {} + - fi - - if [ "$$is_temporary" = "false" ] && [ -n "$$(ls -A $$dir)" ]; then - echo "Path [$$dir] is not empty, skipping..." - return 0 - fi - - echo "Current Ownership and Permissions on [$$dir]:" - echo "chown: $$(stat -c "%u %g" "$$dir")" - echo "chmod: $$(stat -c "%a" "$$dir")" - - if [ "$$mode" = "always" ]; then - fix_owner="true" - fix_perms="true" - fi - - if [ "$$mode" = "check" ]; then - if [ $$(stat -c %u "$$dir") -eq $$uid ] && [ $$(stat -c %g "$$dir") -eq $$gid ]; then - echo "Ownership is correct. Skipping..." - fix_owner="false" - else - echo "Ownership is incorrect. Fixing..." - fix_owner="true" - fi - - if [ "$$chmod" = "false" ]; then - echo "Skipping permissions check, chmod is false" - elif [ -n "$$chmod" ]; then - if [ $$(stat -c %a "$$dir") -eq $$chmod ]; then - echo "Permissions are correct. Skipping..." - fix_perms="false" - else - echo "Permissions are incorrect. Fixing..." - fix_perms="true" - fi - fi - fi - - if [ "$$fix_owner" = "true" ]; then - echo "Changing ownership to $$uid:$$gid on: [$$dir]" - chown -R "$$uid:$$gid" "$$dir" - echo "Finished changing ownership" - echo "Ownership after changes:" - stat -c "%u %g" "$$dir" - fi - - if [ -n "$$chmod" ] && [ "$$fix_perms" = "true" ]; then - echo "Changing permissions to $$chmod on: [$$dir]" - chmod -R "$$chmod" "$$dir" - echo "Finished changing permissions" - echo "Permissions after changes:" - stat -c "%a" "$$dir" - fi -} -""" diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/ports.py b/ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/ports.py deleted file mode 100644 index c895b47a44..0000000000 --- a/ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/ports.py +++ /dev/null @@ -1,42 +0,0 @@ -import ipaddress - -from . import utils - - -def must_valid_port(num: int): - if num < 1 or num > 65535: - utils.throw_error(f"Expected a valid port number, got [{num}]") - - -def must_valid_ip(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - utils.throw_error(f"Expected a valid IP address, got [{ip}]") - - -def must_valid_protocol(protocol: str): - if protocol not in ["tcp", "udp"]: - utils.throw_error(f"Expected a valid protocol, got [{protocol}]") - - -def must_valid_mode(mode: str): - if mode not in ["ingress", "host"]: - utils.throw_error(f"Expected a valid mode, got [{mode}]") - - -def get_port(port=None): - port = port or {} - must_valid_port(port["published"]) - must_valid_port(port["target"]) - must_valid_ip(port.get("host_ip", "0.0.0.0")) - must_valid_protocol(port.get("protocol", "tcp")) - must_valid_mode(port.get("mode", "ingress")) - - return { - "target": port["target"], - "published": port["published"], - "protocol": port.get("protocol", "tcp"), - "mode": port.get("mode", "ingress"), - "host_ip": port.get("host_ip", "0.0.0.0"), - } diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/postgres.py b/ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/postgres.py deleted file mode 100644 index c5f8275454..0000000000 --- a/ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/postgres.py +++ /dev/null @@ -1,77 +0,0 @@ -from . import utils -from .security import get_caps, get_sec_opts -from .network import dns_opts -from .healthchecks import pg_test, check_health -from .resources import resources - - -def pg_url(variant, host, user, password, dbname, port=5432): - if not host: - utils.throw_error("Expected [host] to be set") - if not user: - utils.throw_error("Expected [user] to be set") - if not password: - utils.throw_error("Expected [password] to be set") - if not dbname: - utils.throw_error("Expected [dbname] to be set") - - if variant == "postgresql": - return f"postgresql://{user}:{password}@{host}:{port}/{dbname}?sslmode=disable" - elif variant == "postgres": - return f"postgres://{user}:{password}@{host}:{port}/{dbname}?sslmode=disable" - else: - utils.throw_error( - f"Expected [variant] to be one of [postgresql, postgres], got [{variant}]" - ) - - -def pg_env(user, password, dbname, port=5432): - if not user: - utils.throw_error("Expected [user] to be set for postgres") - if not password: - utils.throw_error("Expected [password] to be set for postgres") - if not dbname: - utils.throw_error("Expected [dbname] to be set for postgres") - return { - "POSTGRES_USER": user, - "POSTGRES_PASSWORD": utils.escape_dollar(password), - "POSTGRES_DB": dbname, - "POSTGRES_PORT": port, - } - - -def pg_container(data={}): - req_keys = ["db_user", "db_password", "db_name", "volumes", "resources"] - for key in req_keys: - if not data.get(key): - utils.throw_error(f"Expected [{key}] to be set for postgres") - - pg_user = data["db_user"] - pg_password = data["db_password"] - pg_dbname = data["db_name"] - pg_port = data.get("port", 5432) - depends = data.get("depends_on", {}) - depends_on = {} - for key in depends: - depends_on[key] = { - "condition": depends[key].get("condition", "service_completed_successfully") - } - - return { - "image": f"{data.get('image', 'postgres:15')}", - "user": f"{data.get('user', '999')}:{data.get('group', '999')}", - "restart": "unless-stopped", - "cap_drop": get_caps()["drop"], - "security_opt": get_sec_opts(), - **({"dns_opts": dns_opts(data["dns_opts"])} if data.get("dns_opts") else {}), - "healthcheck": check_health(pg_test(user=pg_user, db=pg_dbname)), - "environment": pg_env( - user=pg_user, - password=pg_password, - dbname=pg_dbname, - port=pg_port, - ), - "volumes": data["volumes"], - "depends_on": depends_on, - "deploy": {"resources": resources(data["resources"])}, - } diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/redis.py b/ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/redis.py deleted file mode 100644 index 2356d0e06c..0000000000 --- a/ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/redis.py +++ /dev/null @@ -1,49 +0,0 @@ -from . import utils -from .security import get_caps, get_sec_opts -from .network import dns_opts -from .healthchecks import redis_test, check_health -from .resources import resources - - -def redis_container(data={}): - req_keys = ["password", "volumes", "resources"] - for key in req_keys: - if not data.get(key): - utils.throw_error(f"Expected [{key}] to be set for postgres") - - redis_password = data["password"] - redis_port = data.get("port", 6379) - depends = data.get("depends_on", {}) - depends_on = {} - for key in depends: - depends_on[key] = { - "condition": depends[key].get("condition", "service_completed_successfully") - } - - return { - "image": f"{data.get('image', 'bitnami/redis:7.0.11')}", - "user": f"{data.get('user', '1001')}:{data.get('group', '0')}", - "restart": "unless-stopped", - "cap_drop": get_caps()["drop"], - "security_opt": get_sec_opts(), - **({"dns_opts": dns_opts(data["dns_opts"])} if data.get("dns_opts") else {}), - "healthcheck": check_health(redis_test(config={"port": redis_port})), - "environment": redis_env( - password=redis_password, - port=redis_port, - ), - "volumes": data["volumes"], - "depends_on": depends_on, - "deploy": {"resources": resources(data["resources"])}, - } - - -def redis_env(password, port=6379): - if not password: - utils.throw_error("Expected [password] to be set for redis") - - return { - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": utils.escape_dollar(password), - "REDIS_PORT_NUMBER": port, - } diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/resources.py b/ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/resources.py deleted file mode 100644 index d3235fd565..0000000000 --- a/ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/resources.py +++ /dev/null @@ -1,101 +0,0 @@ -import re - -from . import utils - - -def resources(resources, disable_resource_limits=False): - gpus = resources.get("gpus", {}) - cpus = str(resources.get("limits", {}).get("cpus", 2.0)) - memory = str(resources.get("limits", {}).get("memory", 4096)) - if not re.match(r"^[1-9][0-9]*(\.[0-9]+)?$", cpus): - utils.throw_error(f"Expected cpus to be a number or a float, got [{cpus}]") - if not re.match(r"^[1-9][0-9]*$", memory): - raise ValueError(f"Expected memory to be a number, got [{memory}]") - - result = { - "limits": {"cpus": cpus, "memory": f"{memory}M"}, - "reservations": {"devices": []}, - } - - if gpus: - gpu_result = get_nvidia_gpus_reservations(gpus) - if gpu_result: - # Appending to devices, as we can later extend this to support other types of devices. Eg. TPUs. - result["reservations"]["devices"].append(get_nvidia_gpus_reservations(gpus)) - - # Docker does not like empty "things" all around. - if not result["reservations"]["devices"]: - del result["reservations"] - - if disable_resource_limits: - del result["limits"] - - return result - - -def get_nvidia_gpus_reservations(gpus: dict) -> dict: - """ - Input: - { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - """ - if not gpus: - return {} - - device_ids = [] - for pci, gpu in gpus.get("nvidia_gpu_selection", {}).items(): - if gpu["use_gpu"]: - if not gpu.get("uuid"): - utils.throw_error( - "Expected [uuid] to be set for GPU in" - f"slot [{pci}] in [nvidia_gpu_selection]" - ) - device_ids.append(gpu["uuid"]) - - if not device_ids: - return {} - - return { - "capabilities": ["gpu"], - "driver": "nvidia", - "device_ids": device_ids, - } - - -disallowed_devices = ["/dev/dri"] - - -# Returns the top level devices list -# Accepting other_devices to allow manually adding devices -# directly to the list. (Eg sound devices) -def get_devices(resources: dict, other_devices: list = []) -> list: - devices = [] - if resources.get("gpus", {}).get("use_all_gpus", False): - devices.append("/dev/dri:/dev/dri") - - added_host_devices: list = [] - for device in other_devices: - host_device = device.get("host_device", "").rstrip("/") - container_device = device.get("container_device", "") or host_device - if not host_device: - utils.throw_error(f"Expected [host_device] to be set for device [{device}]") - if not utils.valid_path(host_device): - utils.throw_error( - f"Expected [host_device] to be a valid path for device [{device}]" - ) - if host_device in disallowed_devices: - utils.throw_error( - f"Device [{host_device}] is not allowed to be manually added." - ) - if host_device in added_host_devices: - utils.throw_error( - f"Expected devices to be unique, but [{host_device}] was already added." - ) - devices.append(f"{host_device}:{container_device}") - added_host_devices.append(host_device) - - return devices diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/security.py b/ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/security.py deleted file mode 100644 index b67668b51b..0000000000 --- a/ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/security.py +++ /dev/null @@ -1,34 +0,0 @@ -from base64 import b64encode - -from . import utils - - -def get_caps(add=None, drop=None): - add = add or [] - drop = drop or ["ALL"] - result = {"drop": drop} - if add: - result["add"] = add - return result - - -def get_sec_opts(add=None, remove=None): - add = add or [] - remove = remove or [] - result = ["no-new-privileges"] - for opt in add: - if opt not in result: - result.append(opt) - for opt in remove: - if opt in result: - result.remove(opt) - return result - - -def htpasswd(username, password): - hashed = utils.bcrypt_hash(password) - return username + ":" + hashed - - -def basic_auth(username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/storage.py b/ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/storage.py deleted file mode 100644 index de09ba0c3e..0000000000 --- a/ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/storage.py +++ /dev/null @@ -1,370 +0,0 @@ -import re -import json -import hashlib - -from . import utils - - -BIND_TYPES = ["host_path", "ix_volume"] -VOL_TYPES = ["volume", "nfs", "cifs", "temporary"] -ALL_TYPES = BIND_TYPES + VOL_TYPES + ["tmpfs", "anonymous"] -PROPAGATION_TYPES = ["shared", "slave", "private", "rshared", "rslave", "rprivate"] - - -def _get_name_for_temporary(data): - if not data.get("mount_path"): - utils.throw_error("Expected [mount_path] to be set for temporary volume") - - return data["mount_path"].lstrip("/").lower().replace("/", "_").replace(".", "_").replace(" ", "_") - - -# Returns a volume mount object (Used in container's "volumes" level) -def vol_mount(data, values=None): - values = values or {} - ix_volumes = values.get("ix_volumes") or [] - vol_type = _get_docker_vol_type(data) - - volume = { - "type": vol_type, - "target": utils.valid_path(data.get("mount_path", "")), - "read_only": data.get("read_only", False), - } - if vol_type == "bind": # Default create_host_path is true in short-syntax - volume.update(_get_bind_vol_config(data, values, ix_volumes)) - elif vol_type == "volume": - volume.update(_get_volume_vol_config(data)) - elif vol_type == "tmpfs": - volume.update(_get_tmpfs_vol_config(data)) - elif vol_type == "temporary": - volume["type"] = "volume" - volume.update(_get_volume_vol_config(data)) - elif vol_type == "anonymous": - volume["type"] = "volume" - volume.update(_get_anonymous_vol_config(data)) - - return volume - - -def storage_item(data, values=None, perm_opts=None): - values = values or {} - perm_opts = perm_opts or {} - if data.get("type") == "temporary": - data.update({"volume_name": _get_name_for_temporary(data)}) - return { - "vol_mount": vol_mount(data, values), - "vol": vol(data), - "perms_item": perms_item(data, values, perm_opts) if perm_opts else {}, - } - - -def perms_item(data, values=None, opts=None): - opts = opts or {} - values = values or {} - vol_type = data.get("type", "") - - # Temp volumes are always auto permissions - if vol_type == "temporary": - data.update({"auto_permissions": True}) - - # If its ix_volume, we need to set auto permissions - if vol_type == "ix_volume": - data.update({"auto_permissions": True}) - - if not data.get("auto_permissions"): - return {} - - if vol_type == "host_path": - if data.get("host_path_config", {}).get("acl_enable", False): - return {} - if vol_type == "ix_volume": - if data.get("ix_volume_config", {}).get("acl_enable", False): - return {} - - req_keys = ["mount_path", "mode", "uid", "gid"] - for key in req_keys: - if opts.get(key, None) is None: - utils.throw_error(f"Expected opts passed to [perms_item] to have [{key}] key") - - data.update({"mount_path": opts["mount_path"]}) - volume_mount = vol_mount(data, values) - # For perms volume mount, always set read_only to false - volume_mount.update({"read_only": False}) - - return { - "vol_mount": volume_mount, - "perm_dir": { - "dir": volume_mount["target"], - "mode": opts["mode"], - "uid": opts["uid"], - "gid": opts["gid"], - "chmod": opts.get("chmod", "false"), - "is_temporary": data["type"] == "temporary", - }, - } - - -def create_host_path_default(values): - """ - By default, do not create host path for bind mounts if it does not exist. - If the ix_context is missing, we are either in local dev or CI. - We should create the host path by default there to ease development. - The _magic_ "dev_mode" flag is added so we can also toggle this behavior - in CI, while we are also using ix_context for other tests. - """ - ix_ctx = values.get("ix_context", {}) - if not ix_ctx: - return True - if "dev_mode" in ix_ctx: - return ix_ctx["dev_mode"] - return False - - -def _get_bind_vol_config(data, values, ix_volumes=None): - ix_volumes = ix_volumes or [] - path = host_path(data, ix_volumes) - if data.get("propagation", "rprivate") not in PROPAGATION_TYPES: - utils.throw_error( - f"Expected [propagation] to be one of [{', '.join(PROPAGATION_TYPES)}], got [{data['propagation']}]" - ) - - # https://docs.docker.com/storage/bind-mounts/#configure-bind-propagation - return { - "source": path, - "bind": { - "create_host_path": data.get("host_path_config", {}).get( - "create_host_path", create_host_path_default(values) - ), - "propagation": _get_valid_propagation(data), - }, - } - - -def _get_volume_vol_config(data): - if data.get("type") in ["nfs", "cifs"]: - if data.get("volume_name"): - utils.throw_error("Expected [volume_name] to be empty for [nfs, cifs] type") - data.update({"volume_name": _get_name_for_external_volume(data)}) - if not data.get("volume_name"): - utils.throw_error("Expected [volume_name] to be set for [volume] type") - - return {"source": data["volume_name"], "volume": _process_volume_config(data)} - - -def _get_anonymous_vol_config(data): - return {"volume": _process_volume_config(data)} - - -mode_regex = re.compile(r"^0[0-7]{3}$") - - -def _get_tmpfs_vol_config(data): - tmpfs = {} - config = data.get("tmpfs_config", {}) - - if config.get("size"): - if not isinstance(config["size"], int): - utils.throw_error("Expected [size] to be an integer for [tmpfs] type") - if not config["size"] > 0: - utils.throw_error("Expected [size] to be greater than 0 for [tmpfs] type") - # Convert Mebibytes to Bytes - tmpfs.update({"size": config["size"] * 1024 * 1024}) - - if config.get("mode"): - if not mode_regex.match(str(config["mode"])): - utils.throw_error(f"Expected [mode] to be a octal string for [tmpfs] type, got [{config['mode']}]") - tmpfs.update({"mode": int(config["mode"], 8)}) - - return {"tmpfs": tmpfs} - - -# We generate a unique name for the volume based on the config -# Docker will not update any volume after creation. This is to ensure -# that changing any value (eg server address) in the config will result in a new volume -def _get_name_for_external_volume(data): - config_hash = hashlib.sha256(json.dumps(data).encode("utf-8")).hexdigest() - return f"{data['type']}_{config_hash}" - - -# Returns a volume object (Used in top "volumes" level) -def vol(data): - if not data or _get_docker_vol_type(data) != "volume": - return {} - - if not data.get("volume_name"): - utils.throw_error("Expected [volume_name] to be set for [volume] type") - - if data["type"] == "nfs": - return {data["volume_name"]: _process_nfs(data)} - elif data["type"] == "cifs": - return {data["volume_name"]: _process_cifs(data)} - else: - return {data["volume_name"]: {}} - - -def _is_host_path(data): - return data.get("type") == "host_path" - - -def _get_valid_propagation(data): - if not data.get("propagation"): - return "rprivate" - if not data["propagation"] in PROPAGATION_TYPES: - utils.throw_error( - f"Expected [propagation] to be one of [{', '.join(PROPAGATION_TYPES)}], got [{data['propagation']}]" - ) - return data["propagation"] - - -def _is_ix_volume(data): - return data.get("type") == "ix_volume" - - -# Returns the host path for a for either a host_path or ix_volume -def host_path(data, ix_volumes=None): - ix_volumes = ix_volumes or [] - path = "" - if _is_host_path(data): - path = _process_host_path_config(data) - elif _is_ix_volume(data): - path = _process_ix_volume_config(data, ix_volumes) - else: - utils.throw_error( - f"Expected [host_path()] to be called only for types [host_path, ix_volume], got [{data['type']}]" - ) - - return utils.valid_path(path) - - -# Returns the type of storage as used in docker-compose -def _get_docker_vol_type(data): - if not data.get("type"): - utils.throw_error("Expected [type] to be set for storage") - - if data["type"] not in ALL_TYPES: - utils.throw_error(f"Expected storage [type] to be one of {ALL_TYPES}, got [{data['type']}]") - - if data["type"] in BIND_TYPES: - return "bind" - elif data["type"] in VOL_TYPES: - return "volume" - else: - return data["type"] - - -def _process_host_path_config(data): - if data.get("host_path_config", {}).get("acl_enable", False): - if not data["host_path_config"].get("acl", {}).get("path"): - utils.throw_error("Expected [host_path_config.acl.path] to be set for [host_path] type with ACL enabled") - return data["host_path_config"]["acl"]["path"] - - if not data.get("host_path_config", {}).get("path"): - utils.throw_error("Expected [host_path_config.path] to be set for [host_path] type") - - return data["host_path_config"]["path"] - - -def _process_volume_config(data): - return {"nocopy": data.get("volume_config", {}).get("nocopy", False)} - - -def _process_ix_volume_config(data, ix_volumes): - path = "" - if not data.get("ix_volume_config", {}).get("dataset_name"): - utils.throw_error("Expected [ix_volume_config.dataset_name] to be set for [ix_volume] type") - - if not ix_volumes: - utils.throw_error("Expected [ix_volumes] to be set for [ix_volume] type") - - ds = data["ix_volume_config"]["dataset_name"] - path = ix_volumes.get(ds, None) - if not path: - utils.throw_error(f"Expected the key [{ds}] to be set in [ix_volumes]") - - return path - - -# Constructs a volume object for a cifs type -def _process_cifs(data): - if not data.get("cifs_config"): - utils.throw_error("Expected [cifs_config] to be set for [cifs] type") - - required_keys = ["server", "path", "username", "password"] - for key in required_keys: - if not data["cifs_config"].get(key): - utils.throw_error(f"Expected [{key}] to be set for [cifs] type") - - opts = [ - "noperm", - f"user={data['cifs_config']['username']}", - f"password={data['cifs_config']['password']}", - ] - if data["cifs_config"].get("domain"): - opts.append(f'domain={data["cifs_config"]["domain"]}') - - if data["cifs_config"].get("options"): - if not isinstance(data["cifs_config"]["options"], list): - utils.throw_error("Expected [cifs_config.options] to be a list for [cifs] type") - - disallowed_opts = ["user", "password", "domain", "noperm"] - for opt in data["cifs_config"]["options"]: - if not isinstance(opt, str): - utils.throw_error("Expected [cifs_config.options] to be a list of strings for [cifs] type") - - key = opt.split("=")[0] - for disallowed in disallowed_opts: - if key == disallowed: - utils.throw_error( - f"Expected [cifs_config.options] to not start with [{disallowed}] for [cifs] type" - ) - - opts.append(opt) - - server = data["cifs_config"]["server"].lstrip("/") - path = data["cifs_config"]["path"].strip("/") - volume = { - "driver_opts": { - "type": "cifs", - "device": f"//{server}/{path}", - "o": f"{','.join([utils.escape_dollar(opt) for opt in opts])}", - }, - } - - return volume - - -# Constructs a volume object for a nfs type -def _process_nfs(data): - if not data.get("nfs_config"): - utils.throw_error("Expected [nfs_config] to be set for [nfs] type") - - required_keys = ["server", "path"] - for key in required_keys: - if not data["nfs_config"].get(key): - utils.throw_error(f"Expected [{key}] to be set for [nfs] type") - - opts = [f"addr={data['nfs_config']['server']}"] - if data["nfs_config"].get("options"): - if not isinstance(data["nfs_config"]["options"], list): - utils.throw_error("Expected [nfs_config.options] to be a list for [nfs] type") - - disallowed_opts = ["addr"] - for opt in data["nfs_config"]["options"]: - if not isinstance(opt, str): - utils.throw_error("Expected [nfs_config.options] to be a list of strings for [nfs] type") - - key = opt.split("=")[0] - for disallowed in disallowed_opts: - if key == disallowed: - utils.throw_error(f"Expected [nfs_config.options] to not start with [{disallowed}] for [nfs] type") - - opts.append(opt) - - volume = { - "driver_opts": { - "type": "nfs", - "device": f":{data['nfs_config']['path']}", - "o": f"{','.join([utils.escape_dollar(opt) for opt in opts])}", - }, - } - - return volume diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/utils.py b/ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/utils.py deleted file mode 100644 index 8a7c0815c6..0000000000 --- a/ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/utils.py +++ /dev/null @@ -1,124 +0,0 @@ -import hashlib -import secrets -import bcrypt -import sys -import re - -from . import security - - -class TemplateException(Exception): - pass - - -def throw_error(message): - # When throwing a known error, hide the traceback - # This is because the error is also shown in the UI - # and having a traceback makes it hard for user to read - sys.tracebacklimit = 0 - raise TemplateException(message) - - -def secure_string(length): - return secrets.token_urlsafe(length) - - -def basic_auth_header(username, password): - return f"Basic {security.basic_auth(username, password)}" - - -def bcrypt_hash(password): - return bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - - -def match_regex(value, regex): - if not re.match(regex, value): - return False - return True - - -def must_match_regex(value, regex): - if not match_regex(value, regex): - throw_error(f"Expected [{value}] to match [{regex}]") - return value - - -def merge_dicts(*dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - -# Basic validation for a path (Expand later) -def valid_path(path=""): - if not path.startswith("/"): - throw_error(f"Expected path [{path}] to start with /") - - # There is no reason to allow / as a path, either on host or in a container - if path == "/": - throw_error(f"Expected path [{path}] to not be /") - - return path - - -def camel_case(string): - return string.title() - - -def is_boolean(string): - return string.lower() in ["true", "false"] - - -def is_number(string): - try: - float(string) - return True - except ValueError: - return False - - -def get_image(images={}, name=""): - if not images: - throw_error("Expected [images] to be set") - if name not in images: - throw_error(f"Expected [images.{name}] to be set") - if not images[name].get("repository") or not images[name].get("tag"): - throw_error(f"Expected [images.{name}.repository] and [images.{name}.tag] to be set") - - return f"{images[name]['repository']}:{images[name]['tag']}" - - -def hash_data(data=""): - if not data: - throw_error("Expected [data] to be set") - return hashlib.sha256(data.encode("utf-8")).hexdigest() - - -def get_image_with_hashed_data(images={}, name="", data=""): - return f"ix-{get_image(images, name)}-{hash_data(data)}" - - -def copy_dict(dict): - return dict.copy() - - -def escape_dollar(text: str) -> str: - return text.replace("$", "$$") - - -def auto_cast(value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/__init__.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/__init__.py similarity index 100% rename from ix-dev/enterprise/syncthing/templates/library/base_v1_1_7/__init__.py rename to ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/__init__.py diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/configs.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/configs.py similarity index 100% rename from ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/configs.py rename to ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/configs.py diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/container.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/container.py similarity index 100% rename from ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/container.py rename to ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/container.py diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/depends.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/depends.py similarity index 100% rename from ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/depends.py rename to ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/depends.py diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/deploy.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/deploy.py similarity index 100% rename from ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/deploy.py rename to ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/deploy.py diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/deps.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/deps.py similarity index 100% rename from ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/deps.py rename to ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/deps.py diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/device.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/device.py similarity index 100% rename from ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/device.py rename to ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/device.py diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/devices.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/devices.py similarity index 100% rename from ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/devices.py rename to ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/devices.py diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/dns.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/dns.py similarity index 100% rename from ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/dns.py rename to ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/dns.py diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/environment.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/environment.py similarity index 100% rename from ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/environment.py rename to ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/environment.py diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/error.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/error.py similarity index 100% rename from ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/error.py rename to ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/error.py diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/formatter.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/formatter.py similarity index 100% rename from ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/formatter.py rename to ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/formatter.py diff --git a/library/2.0.31/functions.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/functions.py similarity index 99% rename from library/2.0.31/functions.py rename to ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/functions.py index 47a2c1233a..7d082d8c46 100644 --- a/library/2.0.31/functions.py +++ b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/functions.py @@ -1,4 +1,5 @@ import re +import copy import bcrypt import secrets from base64 import b64encode @@ -79,7 +80,7 @@ def _is_number(self, string): return False def _copy_dict(self, dict): - return dict.copy() + return copy.deepcopy(dict) def _merge_dicts(self, *dicts): merged_dict = {} diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/healthcheck.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/healthcheck.py similarity index 100% rename from ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/healthcheck.py rename to ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/healthcheck.py diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/labels.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/labels.py similarity index 100% rename from ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/labels.py rename to ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/labels.py diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/notes.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/notes.py similarity index 100% rename from ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/notes.py rename to ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/notes.py diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/portal.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/portal.py similarity index 100% rename from ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/portal.py rename to ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/portal.py diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/portals.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/portals.py similarity index 100% rename from ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/portals.py rename to ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/portals.py diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/ports.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/ports.py similarity index 100% rename from ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/ports.py rename to ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/ports.py diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/render.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/render.py similarity index 100% rename from ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/render.py rename to ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/render.py diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/resources.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/resources.py similarity index 100% rename from ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/resources.py rename to ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/resources.py diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/restart.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/restart.py similarity index 100% rename from ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/restart.py rename to ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/restart.py diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/storage.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/storage.py similarity index 100% rename from ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/storage.py rename to ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/storage.py diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/sysctls.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/sysctls.py similarity index 100% rename from ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/sysctls.py rename to ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/sysctls.py diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/__init__.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/tests/__init__.py similarity index 100% rename from ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/__init__.py rename to ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/tests/__init__.py diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_build_image.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/tests/test_build_image.py similarity index 100% rename from ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_build_image.py rename to ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/tests/test_build_image.py diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_configs.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/tests/test_configs.py similarity index 100% rename from ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_configs.py rename to ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/tests/test_configs.py diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_container.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/tests/test_container.py similarity index 100% rename from ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_container.py rename to ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/tests/test_container.py diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_depends.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/tests/test_depends.py similarity index 100% rename from ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_depends.py rename to ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/tests/test_depends.py diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_deps.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/tests/test_deps.py similarity index 100% rename from ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_deps.py rename to ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/tests/test_deps.py diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_device.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/tests/test_device.py similarity index 100% rename from ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_device.py rename to ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/tests/test_device.py diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_dns.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/tests/test_dns.py similarity index 100% rename from ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_dns.py rename to ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/tests/test_dns.py diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_environment.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/tests/test_environment.py similarity index 100% rename from ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_environment.py rename to ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/tests/test_environment.py diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_formatter.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/tests/test_formatter.py similarity index 100% rename from ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_formatter.py rename to ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/tests/test_formatter.py diff --git a/library/2.0.31/tests/test_functions.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/tests/test_functions.py similarity index 100% rename from library/2.0.31/tests/test_functions.py rename to ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/tests/test_functions.py diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_healthcheck.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/tests/test_healthcheck.py similarity index 100% rename from ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_healthcheck.py rename to ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/tests/test_healthcheck.py diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_labels.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/tests/test_labels.py similarity index 100% rename from ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_labels.py rename to ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/tests/test_labels.py diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_notes.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/tests/test_notes.py similarity index 100% rename from ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_notes.py rename to ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/tests/test_notes.py diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_portal.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/tests/test_portal.py similarity index 100% rename from ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_portal.py rename to ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/tests/test_portal.py diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_ports.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/tests/test_ports.py similarity index 100% rename from ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_ports.py rename to ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/tests/test_ports.py diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_render.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/tests/test_render.py similarity index 100% rename from ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_render.py rename to ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/tests/test_render.py diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_resources.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/tests/test_resources.py similarity index 100% rename from ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_resources.py rename to ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/tests/test_resources.py diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_restart.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/tests/test_restart.py similarity index 100% rename from ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_restart.py rename to ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/tests/test_restart.py diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_sysctls.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/tests/test_sysctls.py similarity index 100% rename from ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_sysctls.py rename to ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/tests/test_sysctls.py diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_volumes.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/tests/test_volumes.py similarity index 100% rename from ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_volumes.py rename to ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/tests/test_volumes.py diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/validations.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/validations.py similarity index 100% rename from ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/validations.py rename to ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/validations.py diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/volume_mount.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/volume_mount.py similarity index 100% rename from ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/volume_mount.py rename to ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/volume_mount.py diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/volume_mount_types.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/volume_mount_types.py similarity index 100% rename from ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/volume_mount_types.py rename to ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/volume_mount_types.py diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/volume_sources.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/volume_sources.py similarity index 100% rename from ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/volume_sources.py rename to ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/volume_sources.py diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/volume_types.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/volume_types.py similarity index 100% rename from ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/volume_types.py rename to ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/volume_types.py diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/volumes.py b/ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/volumes.py similarity index 100% rename from ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/volumes.py rename to ix-dev/enterprise/syncthing/templates/library/base_v2_0_32/volumes.py diff --git a/ix-dev/enterprise/syncthing/templates/test_values/basic-values.yaml b/ix-dev/enterprise/syncthing/templates/test_values/basic-values.yaml index 64938b7ee3..f32738ca9e 100644 --- a/ix-dev/enterprise/syncthing/templates/test_values/basic-values.yaml +++ b/ix-dev/enterprise/syncthing/templates/test_values/basic-values.yaml @@ -41,14 +41,14 @@ storage: ix_volume_config: dataset_name: data2 create_host_path: true - # Manual test for cifs rendering (migration_mode must add extra options) - - type: cifs - mount_path: /mnt/test/data3 - volume_name: test-data3 - cifs_config: - server: 192.168.1.1 - path: /test - domain: WORKGROUP - username: test - password: test - migration_mode: true + # Manual test for cifs rendering (verify migration_mode adds extra options) + # - type: cifs + # mount_path: /mnt/test/data3 + # volume_name: test-data3 + # cifs_config: + # server: 192.168.1.1 + # path: /test + # domain: WORKGROUP + # username: test + # password: test + # migration_mode: true diff --git a/library/2.0.31/tests/__init__.py b/library/2.0.31/tests/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/__init__.py b/library/2.0.32/__init__.py similarity index 100% rename from ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/__init__.py rename to library/2.0.32/__init__.py diff --git a/library/2.0.31/configs.py b/library/2.0.32/configs.py similarity index 100% rename from library/2.0.31/configs.py rename to library/2.0.32/configs.py diff --git a/library/2.0.31/container.py b/library/2.0.32/container.py similarity index 100% rename from library/2.0.31/container.py rename to library/2.0.32/container.py diff --git a/library/2.0.31/depends.py b/library/2.0.32/depends.py similarity index 100% rename from library/2.0.31/depends.py rename to library/2.0.32/depends.py diff --git a/library/2.0.31/deploy.py b/library/2.0.32/deploy.py similarity index 100% rename from library/2.0.31/deploy.py rename to library/2.0.32/deploy.py diff --git a/library/2.0.31/deps.py b/library/2.0.32/deps.py similarity index 100% rename from library/2.0.31/deps.py rename to library/2.0.32/deps.py diff --git a/library/2.0.31/device.py b/library/2.0.32/device.py similarity index 100% rename from library/2.0.31/device.py rename to library/2.0.32/device.py diff --git a/library/2.0.31/devices.py b/library/2.0.32/devices.py similarity index 100% rename from library/2.0.31/devices.py rename to library/2.0.32/devices.py diff --git a/library/2.0.31/dns.py b/library/2.0.32/dns.py similarity index 100% rename from library/2.0.31/dns.py rename to library/2.0.32/dns.py diff --git a/library/2.0.31/environment.py b/library/2.0.32/environment.py similarity index 100% rename from library/2.0.31/environment.py rename to library/2.0.32/environment.py diff --git a/library/2.0.31/error.py b/library/2.0.32/error.py similarity index 100% rename from library/2.0.31/error.py rename to library/2.0.32/error.py diff --git a/library/2.0.31/formatter.py b/library/2.0.32/formatter.py similarity index 100% rename from library/2.0.31/formatter.py rename to library/2.0.32/formatter.py diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/functions.py b/library/2.0.32/functions.py similarity index 93% rename from ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/functions.py rename to library/2.0.32/functions.py index b0c834ab4b..7d082d8c46 100644 --- a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/functions.py +++ b/library/2.0.32/functions.py @@ -1,4 +1,5 @@ import re +import copy import bcrypt import secrets from base64 import b64encode @@ -79,7 +80,7 @@ def _is_number(self, string): return False def _copy_dict(self, dict): - return dict.copy() + return copy.deepcopy(dict) def _merge_dicts(self, *dicts): merged_dict = {} @@ -98,6 +99,11 @@ def _or_default(self, value, default): return default return value + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + def _get_host_path(self, storage): source_type = storage.get("type", "") if not source_type: @@ -139,4 +145,5 @@ def func_map(self): "disallow_chars": self._disallow_chars, "get_host_path": self._get_host_path, "or_default": self._or_default, + "temp_config": self._temp_config, } diff --git a/library/2.0.31/healthcheck.py b/library/2.0.32/healthcheck.py similarity index 100% rename from library/2.0.31/healthcheck.py rename to library/2.0.32/healthcheck.py diff --git a/library/2.0.31/labels.py b/library/2.0.32/labels.py similarity index 100% rename from library/2.0.31/labels.py rename to library/2.0.32/labels.py diff --git a/library/2.0.31/notes.py b/library/2.0.32/notes.py similarity index 100% rename from library/2.0.31/notes.py rename to library/2.0.32/notes.py diff --git a/library/2.0.31/portal.py b/library/2.0.32/portal.py similarity index 100% rename from library/2.0.31/portal.py rename to library/2.0.32/portal.py diff --git a/library/2.0.31/portals.py b/library/2.0.32/portals.py similarity index 100% rename from library/2.0.31/portals.py rename to library/2.0.32/portals.py diff --git a/library/2.0.31/ports.py b/library/2.0.32/ports.py similarity index 100% rename from library/2.0.31/ports.py rename to library/2.0.32/ports.py diff --git a/library/2.0.31/render.py b/library/2.0.32/render.py similarity index 100% rename from library/2.0.31/render.py rename to library/2.0.32/render.py diff --git a/library/2.0.31/resources.py b/library/2.0.32/resources.py similarity index 100% rename from library/2.0.31/resources.py rename to library/2.0.32/resources.py diff --git a/library/2.0.31/restart.py b/library/2.0.32/restart.py similarity index 100% rename from library/2.0.31/restart.py rename to library/2.0.32/restart.py diff --git a/library/2.0.31/storage.py b/library/2.0.32/storage.py similarity index 100% rename from library/2.0.31/storage.py rename to library/2.0.32/storage.py diff --git a/library/2.0.31/sysctls.py b/library/2.0.32/sysctls.py similarity index 100% rename from library/2.0.31/sysctls.py rename to library/2.0.32/sysctls.py diff --git a/library/2.0.31/__init__.py b/library/2.0.32/tests/__init__.py similarity index 100% rename from library/2.0.31/__init__.py rename to library/2.0.32/tests/__init__.py diff --git a/library/2.0.31/tests/test_build_image.py b/library/2.0.32/tests/test_build_image.py similarity index 100% rename from library/2.0.31/tests/test_build_image.py rename to library/2.0.32/tests/test_build_image.py diff --git a/library/2.0.31/tests/test_configs.py b/library/2.0.32/tests/test_configs.py similarity index 100% rename from library/2.0.31/tests/test_configs.py rename to library/2.0.32/tests/test_configs.py diff --git a/library/2.0.31/tests/test_container.py b/library/2.0.32/tests/test_container.py similarity index 100% rename from library/2.0.31/tests/test_container.py rename to library/2.0.32/tests/test_container.py diff --git a/library/2.0.31/tests/test_depends.py b/library/2.0.32/tests/test_depends.py similarity index 100% rename from library/2.0.31/tests/test_depends.py rename to library/2.0.32/tests/test_depends.py diff --git a/library/2.0.31/tests/test_deps.py b/library/2.0.32/tests/test_deps.py similarity index 100% rename from library/2.0.31/tests/test_deps.py rename to library/2.0.32/tests/test_deps.py diff --git a/library/2.0.31/tests/test_device.py b/library/2.0.32/tests/test_device.py similarity index 100% rename from library/2.0.31/tests/test_device.py rename to library/2.0.32/tests/test_device.py diff --git a/library/2.0.31/tests/test_dns.py b/library/2.0.32/tests/test_dns.py similarity index 100% rename from library/2.0.31/tests/test_dns.py rename to library/2.0.32/tests/test_dns.py diff --git a/library/2.0.31/tests/test_environment.py b/library/2.0.32/tests/test_environment.py similarity index 100% rename from library/2.0.31/tests/test_environment.py rename to library/2.0.32/tests/test_environment.py diff --git a/library/2.0.31/tests/test_formatter.py b/library/2.0.32/tests/test_formatter.py similarity index 100% rename from library/2.0.31/tests/test_formatter.py rename to library/2.0.32/tests/test_formatter.py diff --git a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_functions.py b/library/2.0.32/tests/test_functions.py similarity index 93% rename from ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_functions.py rename to library/2.0.32/tests/test_functions.py index 13d5e49522..0ea3b57d18 100644 --- a/ix-dev/enterprise/syncthing/templates/library/base_v2_0_30/tests/test_functions.py +++ b/library/2.0.32/tests/test_functions.py @@ -66,6 +66,12 @@ def test_funcs(mock_values): {"func": "or_default", "values": [1, None], "expected": 1}, {"func": "or_default", "values": [False, 1], "expected": 1}, {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, ] for test in tests: diff --git a/library/2.0.31/tests/test_healthcheck.py b/library/2.0.32/tests/test_healthcheck.py similarity index 100% rename from library/2.0.31/tests/test_healthcheck.py rename to library/2.0.32/tests/test_healthcheck.py diff --git a/library/2.0.31/tests/test_labels.py b/library/2.0.32/tests/test_labels.py similarity index 100% rename from library/2.0.31/tests/test_labels.py rename to library/2.0.32/tests/test_labels.py diff --git a/library/2.0.31/tests/test_notes.py b/library/2.0.32/tests/test_notes.py similarity index 100% rename from library/2.0.31/tests/test_notes.py rename to library/2.0.32/tests/test_notes.py diff --git a/library/2.0.31/tests/test_portal.py b/library/2.0.32/tests/test_portal.py similarity index 100% rename from library/2.0.31/tests/test_portal.py rename to library/2.0.32/tests/test_portal.py diff --git a/library/2.0.31/tests/test_ports.py b/library/2.0.32/tests/test_ports.py similarity index 100% rename from library/2.0.31/tests/test_ports.py rename to library/2.0.32/tests/test_ports.py diff --git a/library/2.0.31/tests/test_render.py b/library/2.0.32/tests/test_render.py similarity index 100% rename from library/2.0.31/tests/test_render.py rename to library/2.0.32/tests/test_render.py diff --git a/library/2.0.31/tests/test_resources.py b/library/2.0.32/tests/test_resources.py similarity index 100% rename from library/2.0.31/tests/test_resources.py rename to library/2.0.32/tests/test_resources.py diff --git a/library/2.0.31/tests/test_restart.py b/library/2.0.32/tests/test_restart.py similarity index 100% rename from library/2.0.31/tests/test_restart.py rename to library/2.0.32/tests/test_restart.py diff --git a/library/2.0.31/tests/test_sysctls.py b/library/2.0.32/tests/test_sysctls.py similarity index 100% rename from library/2.0.31/tests/test_sysctls.py rename to library/2.0.32/tests/test_sysctls.py diff --git a/library/2.0.31/tests/test_volumes.py b/library/2.0.32/tests/test_volumes.py similarity index 100% rename from library/2.0.31/tests/test_volumes.py rename to library/2.0.32/tests/test_volumes.py diff --git a/library/2.0.31/validations.py b/library/2.0.32/validations.py similarity index 100% rename from library/2.0.31/validations.py rename to library/2.0.32/validations.py diff --git a/library/2.0.31/volume_mount.py b/library/2.0.32/volume_mount.py similarity index 100% rename from library/2.0.31/volume_mount.py rename to library/2.0.32/volume_mount.py diff --git a/library/2.0.31/volume_mount_types.py b/library/2.0.32/volume_mount_types.py similarity index 100% rename from library/2.0.31/volume_mount_types.py rename to library/2.0.32/volume_mount_types.py diff --git a/library/2.0.31/volume_sources.py b/library/2.0.32/volume_sources.py similarity index 100% rename from library/2.0.31/volume_sources.py rename to library/2.0.32/volume_sources.py diff --git a/library/2.0.31/volume_types.py b/library/2.0.32/volume_types.py similarity index 100% rename from library/2.0.31/volume_types.py rename to library/2.0.32/volume_types.py diff --git a/library/2.0.31/volumes.py b/library/2.0.32/volumes.py similarity index 100% rename from library/2.0.31/volumes.py rename to library/2.0.32/volumes.py diff --git a/library/hashes.yaml b/library/hashes.yaml index 80833e1d3f..744eaf23f4 100644 --- a/library/hashes.yaml +++ b/library/hashes.yaml @@ -1,3 +1,3 @@ 0.0.1: f074617a82a86d2a6cc78a4c8a4296fc9d168e456f12713e50c696557b302133 1.1.7: d05e43e25b7dc1736be6cc1efa4b9255368aa346e3e7a4350a38440f29b73186 -2.0.31: e61b4db536830d207e591fea73037ca3f335da01a7e4073bb37392d1ede15873 +2.0.32: 4a0bf69cccda322e191eab36ab81ca6d0c8e5d64a0b2fa117c609804b55b86c6 From 457174dd216e3268fb0f842bb548320fb5cdf0e9 Mon Sep 17 00:00:00 2001 From: Stavros Kois Date: Tue, 3 Dec 2024 18:13:51 +0200 Subject: [PATCH 11/19] add home dir --- ix-dev/enterprise/syncthing/templates/docker-compose.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ix-dev/enterprise/syncthing/templates/docker-compose.yaml b/ix-dev/enterprise/syncthing/templates/docker-compose.yaml index 9e813501da..5be2ae6a58 100644 --- a/ix-dev/enterprise/syncthing/templates/docker-compose.yaml +++ b/ix-dev/enterprise/syncthing/templates/docker-compose.yaml @@ -79,6 +79,8 @@ {% endif %} {% endif %} +{% do c1.add_storage(values.consts.home_path, values.storage.home) %} +{% do config.add_storage(values.consts.home_path, values.storage.home) %} {% for store in values.storage.additional_storage %} {% set new_store = tpl.funcs.copy_dict(store) %} {% if new_store.type == "cifs" and new_store.cifs_config.migration_mode %} From c20b3c456b70caef3f96657f15a8f7dc0c3106af Mon Sep 17 00:00:00 2001 From: Stavros Kois Date: Tue, 3 Dec 2024 18:32:09 +0200 Subject: [PATCH 12/19] fix setup container --- ix-dev/enterprise/syncthing/templates/docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ix-dev/enterprise/syncthing/templates/docker-compose.yaml b/ix-dev/enterprise/syncthing/templates/docker-compose.yaml index 5be2ae6a58..6510fb5c43 100644 --- a/ix-dev/enterprise/syncthing/templates/docker-compose.yaml +++ b/ix-dev/enterprise/syncthing/templates/docker-compose.yaml @@ -28,7 +28,7 @@ {% do config.remove_devices() %} {% do config.deploy.resources.set_profile("medium") %} {% do config.configs.add("setup.sh", setup_script(values, settings), "/setup.sh", "0755") %} -{% do config.set_entrypoint(["/setup.sh"]) %} +{% do config.set_entrypoint(["/bin/sh", "-c", "/setup.sh"]) %} {% do c1.set_user(0, 0) %} {% do config.set_user(0, 0) %} From 0b530047f8e4ab2ad43f93653feda618035cbd37 Mon Sep 17 00:00:00 2001 From: Stavros Kois Date: Tue, 3 Dec 2024 19:35:35 +0200 Subject: [PATCH 13/19] v2-lib: rocm device --- library/{2.0.32 => 2.0.33}/__init__.py | 0 library/{2.0.32 => 2.0.33}/configs.py | 0 library/{2.0.32 => 2.0.33}/container.py | 0 library/{2.0.32 => 2.0.33}/depends.py | 0 library/{2.0.32 => 2.0.33}/deploy.py | 0 library/{2.0.32 => 2.0.33}/deps.py | 0 library/{2.0.32 => 2.0.33}/device.py | 0 library/{2.0.32 => 2.0.33}/devices.py | 1 + library/{2.0.32 => 2.0.33}/dns.py | 0 library/{2.0.32 => 2.0.33}/environment.py | 0 library/{2.0.32 => 2.0.33}/error.py | 0 library/{2.0.32 => 2.0.33}/formatter.py | 0 library/{2.0.32 => 2.0.33}/functions.py | 0 library/{2.0.32 => 2.0.33}/healthcheck.py | 0 library/{2.0.32 => 2.0.33}/labels.py | 0 library/{2.0.32 => 2.0.33}/notes.py | 0 library/{2.0.32 => 2.0.33}/portal.py | 0 library/{2.0.32 => 2.0.33}/portals.py | 0 library/{2.0.32 => 2.0.33}/ports.py | 0 library/{2.0.32 => 2.0.33}/render.py | 0 library/{2.0.32 => 2.0.33}/resources.py | 0 library/{2.0.32 => 2.0.33}/restart.py | 0 library/{2.0.32 => 2.0.33}/storage.py | 0 library/{2.0.32 => 2.0.33}/sysctls.py | 0 library/{2.0.32 => 2.0.33}/tests/__init__.py | 0 library/{2.0.32 => 2.0.33}/tests/test_build_image.py | 0 library/{2.0.32 => 2.0.33}/tests/test_configs.py | 0 library/{2.0.32 => 2.0.33}/tests/test_container.py | 0 library/{2.0.32 => 2.0.33}/tests/test_depends.py | 0 library/{2.0.32 => 2.0.33}/tests/test_deps.py | 0 library/{2.0.32 => 2.0.33}/tests/test_device.py | 2 +- library/{2.0.32 => 2.0.33}/tests/test_dns.py | 0 library/{2.0.32 => 2.0.33}/tests/test_environment.py | 0 library/{2.0.32 => 2.0.33}/tests/test_formatter.py | 0 library/{2.0.32 => 2.0.33}/tests/test_functions.py | 0 library/{2.0.32 => 2.0.33}/tests/test_healthcheck.py | 0 library/{2.0.32 => 2.0.33}/tests/test_labels.py | 0 library/{2.0.32 => 2.0.33}/tests/test_notes.py | 0 library/{2.0.32 => 2.0.33}/tests/test_portal.py | 0 library/{2.0.32 => 2.0.33}/tests/test_ports.py | 0 library/{2.0.32 => 2.0.33}/tests/test_render.py | 0 library/{2.0.32 => 2.0.33}/tests/test_resources.py | 0 library/{2.0.32 => 2.0.33}/tests/test_restart.py | 0 library/{2.0.32 => 2.0.33}/tests/test_sysctls.py | 0 library/{2.0.32 => 2.0.33}/tests/test_volumes.py | 0 library/{2.0.32 => 2.0.33}/validations.py | 2 +- library/{2.0.32 => 2.0.33}/volume_mount.py | 0 library/{2.0.32 => 2.0.33}/volume_mount_types.py | 0 library/{2.0.32 => 2.0.33}/volume_sources.py | 0 library/{2.0.32 => 2.0.33}/volume_types.py | 0 library/{2.0.32 => 2.0.33}/volumes.py | 0 library/hashes.yaml | 2 +- 52 files changed, 4 insertions(+), 3 deletions(-) rename library/{2.0.32 => 2.0.33}/__init__.py (100%) rename library/{2.0.32 => 2.0.33}/configs.py (100%) rename library/{2.0.32 => 2.0.33}/container.py (100%) rename library/{2.0.32 => 2.0.33}/depends.py (100%) rename library/{2.0.32 => 2.0.33}/deploy.py (100%) rename library/{2.0.32 => 2.0.33}/deps.py (100%) rename library/{2.0.32 => 2.0.33}/device.py (100%) rename library/{2.0.32 => 2.0.33}/devices.py (96%) rename library/{2.0.32 => 2.0.33}/dns.py (100%) rename library/{2.0.32 => 2.0.33}/environment.py (100%) rename library/{2.0.32 => 2.0.33}/error.py (100%) rename library/{2.0.32 => 2.0.33}/formatter.py (100%) rename library/{2.0.32 => 2.0.33}/functions.py (100%) rename library/{2.0.32 => 2.0.33}/healthcheck.py (100%) rename library/{2.0.32 => 2.0.33}/labels.py (100%) rename library/{2.0.32 => 2.0.33}/notes.py (100%) rename library/{2.0.32 => 2.0.33}/portal.py (100%) rename library/{2.0.32 => 2.0.33}/portals.py (100%) rename library/{2.0.32 => 2.0.33}/ports.py (100%) rename library/{2.0.32 => 2.0.33}/render.py (100%) rename library/{2.0.32 => 2.0.33}/resources.py (100%) rename library/{2.0.32 => 2.0.33}/restart.py (100%) rename library/{2.0.32 => 2.0.33}/storage.py (100%) rename library/{2.0.32 => 2.0.33}/sysctls.py (100%) rename library/{2.0.32 => 2.0.33}/tests/__init__.py (100%) rename library/{2.0.32 => 2.0.33}/tests/test_build_image.py (100%) rename library/{2.0.32 => 2.0.33}/tests/test_configs.py (100%) rename library/{2.0.32 => 2.0.33}/tests/test_container.py (100%) rename library/{2.0.32 => 2.0.33}/tests/test_depends.py (100%) rename library/{2.0.32 => 2.0.33}/tests/test_deps.py (100%) rename library/{2.0.32 => 2.0.33}/tests/test_device.py (98%) rename library/{2.0.32 => 2.0.33}/tests/test_dns.py (100%) rename library/{2.0.32 => 2.0.33}/tests/test_environment.py (100%) rename library/{2.0.32 => 2.0.33}/tests/test_formatter.py (100%) rename library/{2.0.32 => 2.0.33}/tests/test_functions.py (100%) rename library/{2.0.32 => 2.0.33}/tests/test_healthcheck.py (100%) rename library/{2.0.32 => 2.0.33}/tests/test_labels.py (100%) rename library/{2.0.32 => 2.0.33}/tests/test_notes.py (100%) rename library/{2.0.32 => 2.0.33}/tests/test_portal.py (100%) rename library/{2.0.32 => 2.0.33}/tests/test_ports.py (100%) rename library/{2.0.32 => 2.0.33}/tests/test_render.py (100%) rename library/{2.0.32 => 2.0.33}/tests/test_resources.py (100%) rename library/{2.0.32 => 2.0.33}/tests/test_restart.py (100%) rename library/{2.0.32 => 2.0.33}/tests/test_sysctls.py (100%) rename library/{2.0.32 => 2.0.33}/tests/test_volumes.py (100%) rename library/{2.0.32 => 2.0.33}/validations.py (98%) rename library/{2.0.32 => 2.0.33}/volume_mount.py (100%) rename library/{2.0.32 => 2.0.33}/volume_mount_types.py (100%) rename library/{2.0.32 => 2.0.33}/volume_sources.py (100%) rename library/{2.0.32 => 2.0.33}/volume_types.py (100%) rename library/{2.0.32 => 2.0.33}/volumes.py (100%) diff --git a/library/2.0.32/__init__.py b/library/2.0.33/__init__.py similarity index 100% rename from library/2.0.32/__init__.py rename to library/2.0.33/__init__.py diff --git a/library/2.0.32/configs.py b/library/2.0.33/configs.py similarity index 100% rename from library/2.0.32/configs.py rename to library/2.0.33/configs.py diff --git a/library/2.0.32/container.py b/library/2.0.33/container.py similarity index 100% rename from library/2.0.32/container.py rename to library/2.0.33/container.py diff --git a/library/2.0.32/depends.py b/library/2.0.33/depends.py similarity index 100% rename from library/2.0.32/depends.py rename to library/2.0.33/depends.py diff --git a/library/2.0.32/deploy.py b/library/2.0.33/deploy.py similarity index 100% rename from library/2.0.32/deploy.py rename to library/2.0.33/deploy.py diff --git a/library/2.0.32/deps.py b/library/2.0.33/deps.py similarity index 100% rename from library/2.0.32/deps.py rename to library/2.0.33/deps.py diff --git a/library/2.0.32/device.py b/library/2.0.33/device.py similarity index 100% rename from library/2.0.32/device.py rename to library/2.0.33/device.py diff --git a/library/2.0.32/devices.py b/library/2.0.33/devices.py similarity index 96% rename from library/2.0.32/devices.py rename to library/2.0.33/devices.py index ae22c79d2e..78613bb336 100644 --- a/library/2.0.32/devices.py +++ b/library/2.0.33/devices.py @@ -26,6 +26,7 @@ def _auto_add_devices_from_values(self): resources = self._render_instance.values.get("resources", {}) if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): diff --git a/library/2.0.32/dns.py b/library/2.0.33/dns.py similarity index 100% rename from library/2.0.32/dns.py rename to library/2.0.33/dns.py diff --git a/library/2.0.32/environment.py b/library/2.0.33/environment.py similarity index 100% rename from library/2.0.32/environment.py rename to library/2.0.33/environment.py diff --git a/library/2.0.32/error.py b/library/2.0.33/error.py similarity index 100% rename from library/2.0.32/error.py rename to library/2.0.33/error.py diff --git a/library/2.0.32/formatter.py b/library/2.0.33/formatter.py similarity index 100% rename from library/2.0.32/formatter.py rename to library/2.0.33/formatter.py diff --git a/library/2.0.32/functions.py b/library/2.0.33/functions.py similarity index 100% rename from library/2.0.32/functions.py rename to library/2.0.33/functions.py diff --git a/library/2.0.32/healthcheck.py b/library/2.0.33/healthcheck.py similarity index 100% rename from library/2.0.32/healthcheck.py rename to library/2.0.33/healthcheck.py diff --git a/library/2.0.32/labels.py b/library/2.0.33/labels.py similarity index 100% rename from library/2.0.32/labels.py rename to library/2.0.33/labels.py diff --git a/library/2.0.32/notes.py b/library/2.0.33/notes.py similarity index 100% rename from library/2.0.32/notes.py rename to library/2.0.33/notes.py diff --git a/library/2.0.32/portal.py b/library/2.0.33/portal.py similarity index 100% rename from library/2.0.32/portal.py rename to library/2.0.33/portal.py diff --git a/library/2.0.32/portals.py b/library/2.0.33/portals.py similarity index 100% rename from library/2.0.32/portals.py rename to library/2.0.33/portals.py diff --git a/library/2.0.32/ports.py b/library/2.0.33/ports.py similarity index 100% rename from library/2.0.32/ports.py rename to library/2.0.33/ports.py diff --git a/library/2.0.32/render.py b/library/2.0.33/render.py similarity index 100% rename from library/2.0.32/render.py rename to library/2.0.33/render.py diff --git a/library/2.0.32/resources.py b/library/2.0.33/resources.py similarity index 100% rename from library/2.0.32/resources.py rename to library/2.0.33/resources.py diff --git a/library/2.0.32/restart.py b/library/2.0.33/restart.py similarity index 100% rename from library/2.0.32/restart.py rename to library/2.0.33/restart.py diff --git a/library/2.0.32/storage.py b/library/2.0.33/storage.py similarity index 100% rename from library/2.0.32/storage.py rename to library/2.0.33/storage.py diff --git a/library/2.0.32/sysctls.py b/library/2.0.33/sysctls.py similarity index 100% rename from library/2.0.32/sysctls.py rename to library/2.0.33/sysctls.py diff --git a/library/2.0.32/tests/__init__.py b/library/2.0.33/tests/__init__.py similarity index 100% rename from library/2.0.32/tests/__init__.py rename to library/2.0.33/tests/__init__.py diff --git a/library/2.0.32/tests/test_build_image.py b/library/2.0.33/tests/test_build_image.py similarity index 100% rename from library/2.0.32/tests/test_build_image.py rename to library/2.0.33/tests/test_build_image.py diff --git a/library/2.0.32/tests/test_configs.py b/library/2.0.33/tests/test_configs.py similarity index 100% rename from library/2.0.32/tests/test_configs.py rename to library/2.0.33/tests/test_configs.py diff --git a/library/2.0.32/tests/test_container.py b/library/2.0.33/tests/test_container.py similarity index 100% rename from library/2.0.32/tests/test_container.py rename to library/2.0.33/tests/test_container.py diff --git a/library/2.0.32/tests/test_depends.py b/library/2.0.33/tests/test_depends.py similarity index 100% rename from library/2.0.32/tests/test_depends.py rename to library/2.0.33/tests/test_depends.py diff --git a/library/2.0.32/tests/test_deps.py b/library/2.0.33/tests/test_deps.py similarity index 100% rename from library/2.0.32/tests/test_deps.py rename to library/2.0.33/tests/test_deps.py diff --git a/library/2.0.32/tests/test_device.py b/library/2.0.33/tests/test_device.py similarity index 98% rename from library/2.0.32/tests/test_device.py rename to library/2.0.33/tests/test_device.py index 7455c829f6..41ee86a72a 100644 --- a/library/2.0.32/tests/test_device.py +++ b/library/2.0.33/tests/test_device.py @@ -89,7 +89,7 @@ def test_automatically_add_gpu_devices(mock_values): c1 = render.add_container("test_container", "test_image") c1.healthcheck.disable() output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] assert output["services"]["test_container"]["group_add"] == [44, 107, 568] diff --git a/library/2.0.32/tests/test_dns.py b/library/2.0.33/tests/test_dns.py similarity index 100% rename from library/2.0.32/tests/test_dns.py rename to library/2.0.33/tests/test_dns.py diff --git a/library/2.0.32/tests/test_environment.py b/library/2.0.33/tests/test_environment.py similarity index 100% rename from library/2.0.32/tests/test_environment.py rename to library/2.0.33/tests/test_environment.py diff --git a/library/2.0.32/tests/test_formatter.py b/library/2.0.33/tests/test_formatter.py similarity index 100% rename from library/2.0.32/tests/test_formatter.py rename to library/2.0.33/tests/test_formatter.py diff --git a/library/2.0.32/tests/test_functions.py b/library/2.0.33/tests/test_functions.py similarity index 100% rename from library/2.0.32/tests/test_functions.py rename to library/2.0.33/tests/test_functions.py diff --git a/library/2.0.32/tests/test_healthcheck.py b/library/2.0.33/tests/test_healthcheck.py similarity index 100% rename from library/2.0.32/tests/test_healthcheck.py rename to library/2.0.33/tests/test_healthcheck.py diff --git a/library/2.0.32/tests/test_labels.py b/library/2.0.33/tests/test_labels.py similarity index 100% rename from library/2.0.32/tests/test_labels.py rename to library/2.0.33/tests/test_labels.py diff --git a/library/2.0.32/tests/test_notes.py b/library/2.0.33/tests/test_notes.py similarity index 100% rename from library/2.0.32/tests/test_notes.py rename to library/2.0.33/tests/test_notes.py diff --git a/library/2.0.32/tests/test_portal.py b/library/2.0.33/tests/test_portal.py similarity index 100% rename from library/2.0.32/tests/test_portal.py rename to library/2.0.33/tests/test_portal.py diff --git a/library/2.0.32/tests/test_ports.py b/library/2.0.33/tests/test_ports.py similarity index 100% rename from library/2.0.32/tests/test_ports.py rename to library/2.0.33/tests/test_ports.py diff --git a/library/2.0.32/tests/test_render.py b/library/2.0.33/tests/test_render.py similarity index 100% rename from library/2.0.32/tests/test_render.py rename to library/2.0.33/tests/test_render.py diff --git a/library/2.0.32/tests/test_resources.py b/library/2.0.33/tests/test_resources.py similarity index 100% rename from library/2.0.32/tests/test_resources.py rename to library/2.0.33/tests/test_resources.py diff --git a/library/2.0.32/tests/test_restart.py b/library/2.0.33/tests/test_restart.py similarity index 100% rename from library/2.0.32/tests/test_restart.py rename to library/2.0.33/tests/test_restart.py diff --git a/library/2.0.32/tests/test_sysctls.py b/library/2.0.33/tests/test_sysctls.py similarity index 100% rename from library/2.0.32/tests/test_sysctls.py rename to library/2.0.33/tests/test_sysctls.py diff --git a/library/2.0.32/tests/test_volumes.py b/library/2.0.33/tests/test_volumes.py similarity index 100% rename from library/2.0.32/tests/test_volumes.py rename to library/2.0.33/tests/test_volumes.py diff --git a/library/2.0.32/validations.py b/library/2.0.33/validations.py similarity index 98% rename from library/2.0.32/validations.py rename to library/2.0.33/validations.py index 13f155dfdb..77c1c81396 100644 --- a/library/2.0.32/validations.py +++ b/library/2.0.33/validations.py @@ -139,7 +139,7 @@ def _valid_path_or_raise(path: str): def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb", "/dev/snd"] + disallowed_devices = ["/dev/dri", "/dev/kdf", "/dev/bus/usb", "/dev/snd"] if path in disallowed_devices: raise RenderError(f"Device [{path}] is not allowed to be manually added.") return path diff --git a/library/2.0.32/volume_mount.py b/library/2.0.33/volume_mount.py similarity index 100% rename from library/2.0.32/volume_mount.py rename to library/2.0.33/volume_mount.py diff --git a/library/2.0.32/volume_mount_types.py b/library/2.0.33/volume_mount_types.py similarity index 100% rename from library/2.0.32/volume_mount_types.py rename to library/2.0.33/volume_mount_types.py diff --git a/library/2.0.32/volume_sources.py b/library/2.0.33/volume_sources.py similarity index 100% rename from library/2.0.32/volume_sources.py rename to library/2.0.33/volume_sources.py diff --git a/library/2.0.32/volume_types.py b/library/2.0.33/volume_types.py similarity index 100% rename from library/2.0.32/volume_types.py rename to library/2.0.33/volume_types.py diff --git a/library/2.0.32/volumes.py b/library/2.0.33/volumes.py similarity index 100% rename from library/2.0.32/volumes.py rename to library/2.0.33/volumes.py diff --git a/library/hashes.yaml b/library/hashes.yaml index 744eaf23f4..e838332e5f 100644 --- a/library/hashes.yaml +++ b/library/hashes.yaml @@ -1,3 +1,3 @@ 0.0.1: f074617a82a86d2a6cc78a4c8a4296fc9d168e456f12713e50c696557b302133 1.1.7: d05e43e25b7dc1736be6cc1efa4b9255368aa346e3e7a4350a38440f29b73186 -2.0.32: 4a0bf69cccda322e191eab36ab81ca6d0c8e5d64a0b2fa117c609804b55b86c6 +2.0.33: b30cf75dc79622680a2c740be76709b35ea33cb67852a4bbc1305cbe7c1cf5c3 From 9ac86d75f27673687ea35c268361ad481172cd4c Mon Sep 17 00:00:00 2001 From: Stavros Kois Date: Tue, 3 Dec 2024 19:36:17 +0200 Subject: [PATCH 14/19] fix validation --- library/2.0.33/validations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/2.0.33/validations.py b/library/2.0.33/validations.py index 77c1c81396..b0dea174b7 100644 --- a/library/2.0.33/validations.py +++ b/library/2.0.33/validations.py @@ -139,7 +139,7 @@ def _valid_path_or_raise(path: str): def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/kdf", "/dev/bus/usb", "/dev/snd"] + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] if path in disallowed_devices: raise RenderError(f"Device [{path}] is not allowed to be manually added.") return path From 3d4e6184a14da799e45c3551647f5dd7e1ad391f Mon Sep 17 00:00:00 2001 From: Stavros Kois Date: Wed, 4 Dec 2024 14:54:36 +0200 Subject: [PATCH 15/19] handle rocm --- library/2.0.33/devices.py | 3 ++- library/2.0.33/tests/test_device.py | 10 ++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/library/2.0.33/devices.py b/library/2.0.33/devices.py index 78613bb336..b6139371ee 100644 --- a/library/2.0.33/devices.py +++ b/library/2.0.33/devices.py @@ -26,8 +26,9 @@ def _auto_add_devices_from_values(self): resources = self._render_instance.values.get("resources", {}) if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): # Host device can be mapped to multiple container devices, diff --git a/library/2.0.33/tests/test_device.py b/library/2.0.33/tests/test_device.py index 41ee86a72a..c44437367d 100644 --- a/library/2.0.33/tests/test_device.py +++ b/library/2.0.33/tests/test_device.py @@ -89,6 +89,16 @@ def test_automatically_add_gpu_devices(mock_values): c1 = render.add_container("test_container", "test_image") c1.healthcheck.disable() output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] assert output["services"]["test_container"]["group_add"] == [44, 107, 568] From 824f7dd4a032288c799dad322a90395a6c74691c Mon Sep 17 00:00:00 2001 From: Stavros Kois Date: Wed, 4 Dec 2024 14:55:07 +0200 Subject: [PATCH 16/19] rm old --- library/2.0.32/__init__.py | 0 library/2.0.32/configs.py | 86 --- library/2.0.32/container.py | 317 ----------- library/2.0.32/depends.py | 34 -- library/2.0.32/deploy.py | 24 - library/2.0.32/deps.py | 454 --------------- library/2.0.32/device.py | 31 -- library/2.0.32/devices.py | 66 --- library/2.0.32/dns.py | 79 --- library/2.0.32/environment.py | 109 ---- library/2.0.32/error.py | 4 - library/2.0.32/formatter.py | 26 - library/2.0.32/functions.py | 149 ----- library/2.0.32/healthcheck.py | 193 ------- library/2.0.32/labels.py | 37 -- library/2.0.32/notes.py | 70 --- library/2.0.32/portal.py | 22 - library/2.0.32/portals.py | 28 - library/2.0.32/ports.py | 68 --- library/2.0.32/render.py | 89 --- library/2.0.32/resources.py | 115 ---- library/2.0.32/restart.py | 25 - library/2.0.32/storage.py | 116 ---- library/2.0.32/sysctls.py | 38 -- library/2.0.32/tests/__init__.py | 0 library/2.0.32/tests/test_build_image.py | 49 -- library/2.0.32/tests/test_configs.py | 63 --- library/2.0.32/tests/test_container.py | 324 ----------- library/2.0.32/tests/test_depends.py | 54 -- library/2.0.32/tests/test_deps.py | 380 ------------- library/2.0.32/tests/test_device.py | 121 ---- library/2.0.32/tests/test_dns.py | 64 --- library/2.0.32/tests/test_environment.py | 184 ------- library/2.0.32/tests/test_formatter.py | 13 - library/2.0.32/tests/test_functions.py | 88 --- library/2.0.32/tests/test_healthcheck.py | 187 ------- library/2.0.32/tests/test_labels.py | 88 --- library/2.0.32/tests/test_notes.py | 213 -------- library/2.0.32/tests/test_portal.py | 75 --- library/2.0.32/tests/test_ports.py | 110 ---- library/2.0.32/tests/test_render.py | 37 -- library/2.0.32/tests/test_resources.py | 140 ----- library/2.0.32/tests/test_restart.py | 57 -- library/2.0.32/tests/test_sysctls.py | 62 --- library/2.0.32/tests/test_volumes.py | 666 ----------------------- library/2.0.32/validations.py | 227 -------- library/2.0.32/volume_mount.py | 92 ---- library/2.0.32/volume_mount_types.py | 72 --- library/2.0.32/volume_sources.py | 106 ---- library/2.0.32/volume_types.py | 133 ----- library/2.0.32/volumes.py | 66 --- 51 files changed, 5851 deletions(-) delete mode 100644 library/2.0.32/__init__.py delete mode 100644 library/2.0.32/configs.py delete mode 100644 library/2.0.32/container.py delete mode 100644 library/2.0.32/depends.py delete mode 100644 library/2.0.32/deploy.py delete mode 100644 library/2.0.32/deps.py delete mode 100644 library/2.0.32/device.py delete mode 100644 library/2.0.32/devices.py delete mode 100644 library/2.0.32/dns.py delete mode 100644 library/2.0.32/environment.py delete mode 100644 library/2.0.32/error.py delete mode 100644 library/2.0.32/formatter.py delete mode 100644 library/2.0.32/functions.py delete mode 100644 library/2.0.32/healthcheck.py delete mode 100644 library/2.0.32/labels.py delete mode 100644 library/2.0.32/notes.py delete mode 100644 library/2.0.32/portal.py delete mode 100644 library/2.0.32/portals.py delete mode 100644 library/2.0.32/ports.py delete mode 100644 library/2.0.32/render.py delete mode 100644 library/2.0.32/resources.py delete mode 100644 library/2.0.32/restart.py delete mode 100644 library/2.0.32/storage.py delete mode 100644 library/2.0.32/sysctls.py delete mode 100644 library/2.0.32/tests/__init__.py delete mode 100644 library/2.0.32/tests/test_build_image.py delete mode 100644 library/2.0.32/tests/test_configs.py delete mode 100644 library/2.0.32/tests/test_container.py delete mode 100644 library/2.0.32/tests/test_depends.py delete mode 100644 library/2.0.32/tests/test_deps.py delete mode 100644 library/2.0.32/tests/test_device.py delete mode 100644 library/2.0.32/tests/test_dns.py delete mode 100644 library/2.0.32/tests/test_environment.py delete mode 100644 library/2.0.32/tests/test_formatter.py delete mode 100644 library/2.0.32/tests/test_functions.py delete mode 100644 library/2.0.32/tests/test_healthcheck.py delete mode 100644 library/2.0.32/tests/test_labels.py delete mode 100644 library/2.0.32/tests/test_notes.py delete mode 100644 library/2.0.32/tests/test_portal.py delete mode 100644 library/2.0.32/tests/test_ports.py delete mode 100644 library/2.0.32/tests/test_render.py delete mode 100644 library/2.0.32/tests/test_resources.py delete mode 100644 library/2.0.32/tests/test_restart.py delete mode 100644 library/2.0.32/tests/test_sysctls.py delete mode 100644 library/2.0.32/tests/test_volumes.py delete mode 100644 library/2.0.32/validations.py delete mode 100644 library/2.0.32/volume_mount.py delete mode 100644 library/2.0.32/volume_mount_types.py delete mode 100644 library/2.0.32/volume_sources.py delete mode 100644 library/2.0.32/volume_types.py delete mode 100644 library/2.0.32/volumes.py diff --git a/library/2.0.32/__init__.py b/library/2.0.32/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/library/2.0.32/configs.py b/library/2.0.32/configs.py deleted file mode 100644 index b76f4b169c..0000000000 --- a/library/2.0.32/configs.py +++ /dev/null @@ -1,86 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise - - -class Configs: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._configs: dict[str, dict] = {} - - def add(self, name: str, data: str): - if not isinstance(data, str): - raise RenderError(f"Expected [data] to be a string, got [{type(data)}]") - - if name not in self._configs: - self._configs[name] = {"name": name, "data": data} - return - - if data == self._configs[name]["data"]: - return - - raise RenderError(f"Config [{name}] already added with different data") - - def has_configs(self): - return bool(self._configs) - - def render(self): - return { - c["name"]: {"content": escape_dollar(c["data"])} - for c in sorted(self._configs.values(), key=lambda c: c["name"]) - } - - -class ContainerConfigs: - def __init__(self, render_instance: "Render", configs: Configs): - self._render_instance = render_instance - self.top_level_configs: Configs = configs - self.container_configs: set[ContainerConfig] = set() - - def add(self, name: str, data: str, target: str, mode: str = ""): - self.top_level_configs.add(name, data) - - if target == "": - raise RenderError(f"Expected [target] to be set for config [{name}]") - if mode != "": - mode = valid_octal_mode_or_raise(mode) - - if target in [c.target for c in self.container_configs]: - raise RenderError(f"Target [{target}] already used for another config") - target = valid_fs_path_or_raise(target) - self.container_configs.add(ContainerConfig(self._render_instance, name, target, mode)) - - def has_configs(self): - return bool(self.container_configs) - - def render(self): - return [c.render() for c in sorted(self.container_configs, key=lambda c: c.source)] - - -class ContainerConfig: - def __init__(self, render_instance: "Render", source: str, target: str, mode: str): - self._render_instance = render_instance - self.source = source - self.target = target - self.mode = mode - - def render(self): - result: dict[str, str | int] = { - "source": self.source, - "target": self.target, - } - - if self.mode: - result["mode"] = int(self.mode, 8) - - return result diff --git a/library/2.0.32/container.py b/library/2.0.32/container.py deleted file mode 100644 index a95e76734c..0000000000 --- a/library/2.0.32/container.py +++ /dev/null @@ -1,317 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage - from .sysctls import Sysctls -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - from sysctls import Sysctls - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.sysctls: Sysctls = Sysctls(self._render_instance, self) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - self.add_group(999) - self._storage._add_docker_socket(read_only, mount_path) - - def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): - self._storage._add_tun_device(read_only, mount_path) - - def add_snd_device(self): - self.add_group(29) - self.devices._add_snd_device() - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = f"{self._grace_period}s" - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_gpus(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self.sysctls.has_sysctls(): - result["sysctls"] = self.sysctls.render() - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/library/2.0.32/depends.py b/library/2.0.32/depends.py deleted file mode 100644 index 4e057cf085..0000000000 --- a/library/2.0.32/depends.py +++ /dev/null @@ -1,34 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_depend_condition_or_raise -except ImportError: - from error import RenderError - from validations import valid_depend_condition_or_raise - - -class Depends: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._dependencies: dict[str, str] = {} - - def add_dependency(self, name: str, condition: str): - condition = valid_depend_condition_or_raise(condition) - if name in self._dependencies.keys(): - raise RenderError(f"Dependency [{name}] already added") - if name not in self._render_instance.container_names(): - raise RenderError( - f"Dependency [{name}] not found in defined containers. " - f"Available containers: [{', '.join(self._render_instance.container_names())}]" - ) - self._dependencies[name] = condition - - def has_dependencies(self): - return len(self._dependencies) > 0 - - def render(self): - return {d: {"condition": c} for d, c in self._dependencies.items()} diff --git a/library/2.0.32/deploy.py b/library/2.0.32/deploy.py deleted file mode 100644 index 894dbc643b..0000000000 --- a/library/2.0.32/deploy.py +++ /dev/null @@ -1,24 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .resources import Resources -except ImportError: - from resources import Resources - - -class Deploy: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self.resources: Resources = Resources(self._render_instance) - - def has_deploy(self): - return self.resources.has_resources() - - def render(self): - if self.resources.has_resources(): - return {"resources": self.resources.render()} - - return {} diff --git a/library/2.0.32/deps.py b/library/2.0.32/deps.py deleted file mode 100644 index b3607fa6ab..0000000000 --- a/library/2.0.32/deps.py +++ /dev/null @@ -1,454 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - recursive = action_config.get("recursive", False) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - recursive = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "recursive": recursive, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod, recursive=False): - print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chmod(os.path.join(root, f), int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid, recursive=False): - print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chown(os.path.join(root, f), uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/library/2.0.32/device.py b/library/2.0.32/device.py deleted file mode 100644 index bfe97097cb..0000000000 --- a/library/2.0.32/device.py +++ /dev/null @@ -1,31 +0,0 @@ -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise, allowed_device_or_raise, valid_cgroup_perm_or_raise -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise, allowed_device_or_raise, valid_cgroup_perm_or_raise - - -class Device: - def __init__(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - hd = valid_fs_path_or_raise(host_device.rstrip("/")) - cd = valid_fs_path_or_raise(container_device.rstrip("/")) - if not hd or not cd: - raise RenderError( - "Expected [host_device] and [container_device] to be set. " - f"Got host_device [{host_device}] and container_device [{container_device}]" - ) - - cgroup_perm = valid_cgroup_perm_or_raise(cgroup_perm) - if not allow_disallowed: - hd = allowed_device_or_raise(hd) - - self.cgroup_perm: str = cgroup_perm - self.host_device: str = hd - self.container_device: str = cd - - def render(self): - result = f"{self.host_device}:{self.container_device}" - if self.cgroup_perm: - result += f":{self.cgroup_perm}" - return result diff --git a/library/2.0.32/devices.py b/library/2.0.32/devices.py deleted file mode 100644 index ae22c79d2e..0000000000 --- a/library/2.0.32/devices.py +++ /dev/null @@ -1,66 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def _add_snd_device(self): - self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/library/2.0.32/dns.py b/library/2.0.32/dns.py deleted file mode 100644 index d3ae7b19fa..0000000000 --- a/library/2.0.32/dns.py +++ /dev/null @@ -1,79 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import allowed_dns_opt_or_raise -except ImportError: - from error import RenderError - from validations import allowed_dns_opt_or_raise - - -class Dns: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._dns_options: set[str] = set() - self._dns_searches: set[str] = set() - self._dns_nameservers: set[str] = set() - - self._auto_add_dns_opts_from_values() - self._auto_add_dns_searches_from_values() - self._auto_add_dns_nameservers_from_values() - - def _get_dns_opt_keys(self): - return [self._get_key_from_opt(opt) for opt in self._dns_options] - - def _get_key_from_opt(self, opt): - return opt.split(":")[0] - - def _auto_add_dns_opts_from_values(self): - values = self._render_instance.values - for dns_opt in values.get("network", {}).get("dns_opts", []): - self.add_dns_opt(dns_opt) - - def _auto_add_dns_searches_from_values(self): - values = self._render_instance.values - for dns_search in values.get("network", {}).get("dns_searches", []): - self.add_dns_search(dns_search) - - def _auto_add_dns_nameservers_from_values(self): - values = self._render_instance.values - for dns_nameserver in values.get("network", {}).get("dns_nameservers", []): - self.add_dns_nameserver(dns_nameserver) - - def add_dns_search(self, dns_search): - if dns_search in self._dns_searches: - raise RenderError(f"DNS Search [{dns_search}] already added") - self._dns_searches.add(dns_search) - - def add_dns_nameserver(self, dns_nameserver): - if dns_nameserver in self._dns_nameservers: - raise RenderError(f"DNS Nameserver [{dns_nameserver}] already added") - self._dns_nameservers.add(dns_nameserver) - - def add_dns_opt(self, dns_opt): - # eg attempts:3 - key = allowed_dns_opt_or_raise(self._get_key_from_opt(dns_opt)) - if key in self._get_dns_opt_keys(): - raise RenderError(f"DNS Option [{key}] already added") - self._dns_options.add(dns_opt) - - def has_dns_opts(self): - return len(self._dns_options) > 0 - - def has_dns_searches(self): - return len(self._dns_searches) > 0 - - def has_dns_nameservers(self): - return len(self._dns_nameservers) > 0 - - def render_dns_searches(self): - return sorted(self._dns_searches) - - def render_dns_opts(self): - return sorted(self._dns_options) - - def render_dns_nameservers(self): - return sorted(self._dns_nameservers) diff --git a/library/2.0.32/environment.py b/library/2.0.32/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/library/2.0.32/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/library/2.0.32/error.py b/library/2.0.32/error.py deleted file mode 100644 index aef48d3b02..0000000000 --- a/library/2.0.32/error.py +++ /dev/null @@ -1,4 +0,0 @@ -class RenderError(Exception): - """Base class for exceptions in this module.""" - - pass diff --git a/library/2.0.32/formatter.py b/library/2.0.32/formatter.py deleted file mode 100644 index 24e882f47a..0000000000 --- a/library/2.0.32/formatter.py +++ /dev/null @@ -1,26 +0,0 @@ -import json -import hashlib - - -def escape_dollar(text: str) -> str: - return text.replace("$", "$$") - - -def get_hashed_name_for_volume(prefix: str, config: dict): - config_hash = hashlib.sha256(json.dumps(config).encode("utf-8")).hexdigest() - return f"{prefix}_{config_hash}" - - -def get_hash_with_prefix(prefix: str, data: str): - return f"{prefix}_{hashlib.sha256(data.encode('utf-8')).hexdigest()}" - - -def merge_dicts_no_overwrite(dict1, dict2): - overlapping_keys = dict1.keys() & dict2.keys() - if overlapping_keys: - raise ValueError(f"Merging of dicts failed. Overlapping keys: {overlapping_keys}") - return {**dict1, **dict2} - - -def get_image_with_hashed_data(image: str, data: str): - return get_hash_with_prefix(f"ix-{image}", data) diff --git a/library/2.0.32/functions.py b/library/2.0.32/functions.py deleted file mode 100644 index 7d082d8c46..0000000000 --- a/library/2.0.32/functions.py +++ /dev/null @@ -1,149 +0,0 @@ -import re -import copy -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .volume_sources import HostPathSource, IxVolumeSource -except ImportError: - from error import RenderError - from volume_sources import HostPathSource, IxVolumeSource - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return copy.deepcopy(dict) - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string: str, chars: list[str], key: str): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - return string - - def _or_default(self, value, default): - if not value: - return default - return value - - def _temp_config(self, name): - if not name: - raise RenderError("Expected [name] to be set when calling [temp_config].") - return {"type": "temporary", "volume_config": {"volume_name": name}} - - def _get_host_path(self, storage): - source_type = storage.get("type", "") - if not source_type: - raise RenderError("Expected [type] to be set for volume mounts.") - - match source_type: - case "host_path": - mount_config = storage.get("host_path_config") - if mount_config is None: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - host_source = HostPathSource(self._render_instance, mount_config).get() - return host_source - case "ix_volume": - mount_config = storage.get("ix_volume_config") - if mount_config is None: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - ix_source = IxVolumeSource(self._render_instance, mount_config).get() - return ix_source - case _: - raise RenderError(f"Storage type [{source_type}] does not support host path.") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - "get_host_path": self._get_host_path, - "or_default": self._or_default, - "temp_config": self._temp_config, - } diff --git a/library/2.0.32/healthcheck.py b/library/2.0.32/healthcheck.py deleted file mode 100644 index 36ae5d90aa..0000000000 --- a/library/2.0.32/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/library/2.0.32/labels.py b/library/2.0.32/labels.py deleted file mode 100644 index f1e667ba00..0000000000 --- a/library/2.0.32/labels.py +++ /dev/null @@ -1,37 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar -except ImportError: - from error import RenderError - from formatter import escape_dollar - - -class Labels: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._labels: dict[str, str] = {} - - def add_label(self, key: str, value: str): - if not key: - raise RenderError("Labels must have a key") - - if key.startswith("com.docker.compose"): - raise RenderError(f"Label [{key}] cannot start with [com.docker.compose] as it is reserved") - - if key in self._labels.keys(): - raise RenderError(f"Label [{key}] already added") - - self._labels[key] = escape_dollar(str(value)) - - def has_labels(self) -> bool: - return bool(self._labels) - - def render(self) -> dict[str, str]: - if not self.has_labels(): - return {} - return {label: value for label, value in sorted(self._labels.items())} diff --git a/library/2.0.32/notes.py b/library/2.0.32/notes.py deleted file mode 100644 index 4adc50c3d8..0000000000 --- a/library/2.0.32/notes.py +++ /dev/null @@ -1,70 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - - -class Notes: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._app_name: str = "" - self._warnings: list[str] = [] - self._deprecations: list[str] = [] - self._header: str = "" - self._body: str = "" - self._footer: str = "" - - self._auto_set_app_name() - self._auto_set_header() - self._auto_set_footer() - - def _auto_set_app_name(self): - app_name = self._render_instance.values.get("ix_context", {}).get("app_metadata", {}).get("name", "") - self._app_name = app_name or "" - - def _auto_set_header(self): - head = "# Welcome to TrueNAS SCALE\n\n" - head += f"Thank you for installing {self._app_name}!\n\n" - self._header = head - - def _auto_set_footer(self): - footer = "## Documentation\n\n" - footer += f"Documentation for {self._app_name} can be found at https://www.truenas.com/docs.\n\n" - footer += "## Bug reports\n\n" - footer += "If you find a bug in this app, please file an issue at\n" - footer += "https://ixsystems.atlassian.net or https://github.com/truenas/apps\n\n" - footer += "## Feature requests or improvements\n\n" - footer += "If you find a feature request for this app, please file an issue at\n" - footer += "https://ixsystems.atlassian.net or https://github.com/truenas/apps\n" - self._footer = footer - - def add_warning(self, warning: str): - self._warnings.append(warning) - - def add_deprecation(self, deprecation: str): - self._deprecations.append(deprecation) - - def set_body(self, body: str): - self._body = body - - def render(self): - result = self._header - - if self._warnings: - result += "## Warnings\n\n" - for warning in self._warnings: - result += f"- {warning}\n" - result += "\n" - - if self._deprecations: - result += "## Deprecations\n\n" - for deprecation in self._deprecations: - result += f"- {deprecation}\n" - result += "\n" - - if self._body: - result += self._body.strip() + "\n\n" - - result += self._footer - - return result diff --git a/library/2.0.32/portal.py b/library/2.0.32/portal.py deleted file mode 100644 index cf47163439..0000000000 --- a/library/2.0.32/portal.py +++ /dev/null @@ -1,22 +0,0 @@ -try: - from .validations import valid_portal_scheme_or_raise, valid_http_path_or_raise, valid_port_or_raise -except ImportError: - from validations import valid_portal_scheme_or_raise, valid_http_path_or_raise, valid_port_or_raise - - -class Portal: - def __init__(self, name: str, config: dict): - self._name = name - self._scheme = valid_portal_scheme_or_raise(config.get("scheme", "http")) - self._host = config.get("host", "0.0.0.0") or "0.0.0.0" - self._port = valid_port_or_raise(config.get("port", 0)) - self._path = valid_http_path_or_raise(config.get("path", "/")) - - def render(self): - return { - "name": self._name, - "scheme": self._scheme, - "host": self._host, - "port": self._port, - "path": self._path, - } diff --git a/library/2.0.32/portals.py b/library/2.0.32/portals.py deleted file mode 100644 index e106d231e6..0000000000 --- a/library/2.0.32/portals.py +++ /dev/null @@ -1,28 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .portal import Portal -except ImportError: - from error import RenderError - from portal import Portal - - -class Portals: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._portals: set[Portal] = set() - - def add_portal(self, config: dict): - name = config.get("name", "Web UI") - - if name in [p._name for p in self._portals]: - raise RenderError(f"Portal [{name}] already added") - - self._portals.add(Portal(name, config)) - - def render(self): - return [p.render() for _, p in sorted([(p._name, p) for p in self._portals])] diff --git a/library/2.0.32/ports.py b/library/2.0.32/ports.py deleted file mode 100644 index f11e1481b4..0000000000 --- a/library/2.0.32/ports.py +++ /dev/null @@ -1,68 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_port_protocol_or_raise, - valid_port_mode_or_raise, - valid_ip_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_port_protocol_or_raise, - valid_port_mode_or_raise, - valid_ip_or_raise, - ) - - -class Ports: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._ports: dict[str, dict] = {} - - def add_port(self, host_port: int, container_port: int, config: dict | None = None): - config = config or {} - host_port = valid_port_or_raise(host_port) - container_port = valid_port_or_raise(container_port) - proto = valid_port_protocol_or_raise(config.get("protocol", "tcp")) - mode = valid_port_mode_or_raise(config.get("mode", "ingress")) - host_ip = valid_ip_or_raise(config.get("host_ip", "0.0.0.0")) - - key = f"{host_port}_{host_ip}_{proto}" - if key in self._ports.keys(): - raise RenderError(f"Port [{host_port}/{proto}] already added for [{host_ip}]") - - if host_ip != "0.0.0.0": - # If the port we are adding is not going to use 0.0.0.0 - # Make sure that we don't have already added that port/proto to 0.0.0.0 - search_key = f"{host_port}_0.0.0.0_{proto}" - if search_key in self._ports.keys(): - raise RenderError(f"Cannot bind port [{host_port}/{proto}] to [{host_ip}], already bound to [0.0.0.0]") - elif host_ip == "0.0.0.0": - # If the port we are adding is going to use 0.0.0.0 - # Make sure that we don't have already added that port/proto to a specific ip - for p in self._ports.values(): - if p["published"] == host_port and p["protocol"] == proto: - raise RenderError( - f"Cannot bind port [{host_port}/{proto}] to [{host_ip}], already bound to [{p['host_ip']}]" - ) - - self._ports[key] = { - "published": host_port, - "target": container_port, - "protocol": proto, - "mode": mode, - "host_ip": host_ip, - } - - def has_ports(self): - return len(self._ports) > 0 - - def render(self): - return [config for _, config in sorted(self._ports.items())] diff --git a/library/2.0.32/render.py b/library/2.0.32/render.py deleted file mode 100644 index 9d8fcc28d5..0000000000 --- a/library/2.0.32/render.py +++ /dev/null @@ -1,89 +0,0 @@ -import copy - -try: - from .configs import Configs - from .container import Container - from .deps import Deps - from .error import RenderError - from .functions import Functions - from .notes import Notes - from .portals import Portals - from .volumes import Volumes -except ImportError: - from configs import Configs - from container import Container - from deps import Deps - from error import RenderError - from functions import Functions - from notes import Notes - from portals import Portals - from volumes import Volumes - - -class Render(object): - def __init__(self, values): - self._containers: dict[str, Container] = {} - self.values = values - self._add_images_internal_use() - # Make a copy after we inject the images - self._original_values: dict = copy.deepcopy(self.values) - - self.deps: Deps = Deps(self) - - self.configs = Configs(render_instance=self) - self.funcs = Functions(render_instance=self).func_map() - self.portals: Portals = Portals(render_instance=self) - self.notes: Notes = Notes(render_instance=self) - self.volumes = Volumes(render_instance=self) - - def _add_images_internal_use(self): - if not self.values.get("images"): - self.values["images"] = {} - - if "python_permissions_image" not in self.values["images"]: - self.values["images"]["python_permissions_image"] = {"repository": "python", "tag": "3.13.0-slim-bookworm"} - - def container_names(self): - return list(self._containers.keys()) - - def add_container(self, name: str, image: str): - name = name.strip() - if not name: - raise RenderError("Container name cannot be empty") - container = Container(self, name, image) - if name in self._containers: - raise RenderError(f"Container {name} already exists.") - self._containers[name] = container - return container - - def render(self): - if self.values != self._original_values: - raise RenderError("Values have been modified since the renderer was created.") - - if not self._containers: - raise RenderError("No containers added.") - - result: dict = { - "x-notes": self.notes.render(), - "x-portals": self.portals.render(), - "services": {c._name: c.render() for c in self._containers.values()}, - } - - # Make sure that after services are rendered - # there are no labels that target a non-existent container - # This is to prevent typos - for label in self.values.get("labels", []): - for c in label.get("containers", []): - if c not in self.container_names(): - raise RenderError(f"Label [{label['key']}] references container [{c}] which does not exist") - - if self.volumes.has_volumes(): - result["volumes"] = self.volumes.render() - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - # if self.networks: - # result["networks"] = {...} - - return result diff --git a/library/2.0.32/resources.py b/library/2.0.32/resources.py deleted file mode 100644 index 733f43bb6f..0000000000 --- a/library/2.0.32/resources.py +++ /dev/null @@ -1,115 +0,0 @@ -import re -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -DEFAULT_CPUS = 2.0 -DEFAULT_MEMORY = 4096 - - -class Resources: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._limits: dict = {} - self._reservations: dict = {} - self._nvidia_ids: set[str] = set() - self._auto_add_cpu_from_values() - self._auto_add_memory_from_values() - self._auto_add_gpus_from_values() - - def _set_cpu(self, cpus: Any): - c = str(cpus) - if not re.match(r"^[1-9][0-9]*(\.[0-9]+)?$", c): - raise RenderError(f"Expected cpus to be a number or a float (minimum 1.0), got [{cpus}]") - self._limits.update({"cpus": c}) - - def _set_memory(self, memory: Any): - m = str(memory) - if not re.match(r"^[1-9][0-9]*$", m): - raise RenderError(f"Expected memory to be a number, got [{memory}]") - self._limits.update({"memory": f"{m}M"}) - - def _auto_add_cpu_from_values(self): - resources = self._render_instance.values.get("resources", {}) - self._set_cpu(resources.get("limits", {}).get("cpus", DEFAULT_CPUS)) - - def _auto_add_memory_from_values(self): - resources = self._render_instance.values.get("resources", {}) - self._set_memory(resources.get("limits", {}).get("memory", DEFAULT_MEMORY)) - - def _auto_add_gpus_from_values(self): - resources = self._render_instance.values.get("resources", {}) - gpus = resources.get("gpus", {}).get("nvidia_gpu_selection", {}) - if not gpus: - return - - for pci, gpu in gpus.items(): - if gpu.get("use_gpu", False): - if not gpu.get("uuid"): - raise RenderError(f"Expected [uuid] to be set for GPU in slot [{pci}] in [nvidia_gpu_selection]") - self._nvidia_ids.add(gpu["uuid"]) - - if self._nvidia_ids: - if not self._reservations: - self._reservations["devices"] = [] - self._reservations["devices"].append( - { - "capabilities": ["gpu"], - "driver": "nvidia", - "device_ids": sorted(self._nvidia_ids), - } - ) - - # This is only used on ix-app that we allow - # disabling cpus and memory. GPUs are only added - # if the user has requested them. - def remove_cpus_and_memory(self): - self._limits.pop("cpus", None) - self._limits.pop("memory", None) - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._reservations.pop("devices", None) - - def set_profile(self, profile: str): - cpu, memory = profile_mapping(profile) - self._set_cpu(cpu) - self._set_memory(memory) - - def has_resources(self): - return len(self._limits) > 0 or len(self._reservations) > 0 - - def has_gpus(self): - gpu_devices = [d for d in self._reservations.get("devices", []) if "gpu" in d["capabilities"]] - return len(gpu_devices) > 0 - - def render(self): - result = {} - if self._limits: - result["limits"] = self._limits - if self._reservations: - result["reservations"] = self._reservations - - return result - - -def profile_mapping(profile: str): - profiles = { - "low": (1, 512), - "medium": (2, 1024), - } - - if profile not in profiles: - raise RenderError( - f"Resource profile [{profile}] is not valid. Valid options are: [{', '.join(profiles.keys())}]" - ) - - return profiles[profile] diff --git a/library/2.0.32/restart.py b/library/2.0.32/restart.py deleted file mode 100644 index 2f6281af48..0000000000 --- a/library/2.0.32/restart.py +++ /dev/null @@ -1,25 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .validations import valid_restart_policy_or_raise -except ImportError: - from validations import valid_restart_policy_or_raise - - -class RestartPolicy: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._policy: str = "unless-stopped" - self._maximum_retry_count: int = 0 - - def set_policy(self, policy: str, maximum_retry_count: int = 0): - self._policy = valid_restart_policy_or_raise(policy, maximum_retry_count) - self._maximum_retry_count = maximum_retry_count - - def render(self): - if self._policy == "on-failure" and self._maximum_retry_count > 0: - return f"{self._policy}:{self._maximum_retry_count}" - return self._policy diff --git a/library/2.0.32/storage.py b/library/2.0.32/storage.py deleted file mode 100644 index e697ba902a..0000000000 --- a/library/2.0.32/storage.py +++ /dev/null @@ -1,116 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): - mount_path = valid_fs_path_or_raise(mount_path) - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): - mount_path = valid_fs_path_or_raise(mount_path) - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/library/2.0.32/sysctls.py b/library/2.0.32/sysctls.py deleted file mode 100644 index e6b8469f3b..0000000000 --- a/library/2.0.32/sysctls.py +++ /dev/null @@ -1,38 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from container import Container - -try: - from .error import RenderError - from .validations import valid_sysctl_or_raise -except ImportError: - from error import RenderError - from validations import valid_sysctl_or_raise - - -class Sysctls: - def __init__(self, render_instance: "Render", container_instance: "Container"): - self._render_instance = render_instance - self._container_instance = container_instance - self._sysctls: dict = {} - - def add(self, key: str, value): - key = key.strip() - if not key: - raise RenderError("Sysctls key cannot be empty") - if value is None: - raise RenderError(f"Sysctl [{key}] requires a value") - if key in self._sysctls: - raise RenderError(f"Sysctl [{key}] already added") - self._sysctls[key] = str(value) - - def has_sysctls(self): - return bool(self._sysctls) - - def render(self): - if not self.has_sysctls(): - return {} - host_net = self._container_instance._network_mode == "host" - return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/library/2.0.32/tests/__init__.py b/library/2.0.32/tests/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/library/2.0.32/tests/test_build_image.py b/library/2.0.32/tests/test_build_image.py deleted file mode 100644 index f30c1210ed..0000000000 --- a/library/2.0.32/tests/test_build_image.py +++ /dev/null @@ -1,49 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_build_image_with_from(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.build_image(["FROM test_image"]) - - -def test_build_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.build_image( - [ - "RUN echo hello", - None, - "", - "RUN echo world", - ] - ) - output = render.render() - assert ( - output["services"]["test_container"]["image"] - == "ix-nginx:latest_4a127145ea4c25511707e57005dd0ed457fe2f4932082c8f9faa339a450b6a99" - ) - assert output["services"]["test_container"]["build"] == { - "tags": ["ix-nginx:latest_4a127145ea4c25511707e57005dd0ed457fe2f4932082c8f9faa339a450b6a99"], - "dockerfile_inline": """FROM nginx:latest -RUN echo hello -RUN echo world -""", - } diff --git a/library/2.0.32/tests/test_configs.py b/library/2.0.32/tests/test_configs.py deleted file mode 100644 index 9049e473ea..0000000000 --- a/library/2.0.32/tests/test_configs.py +++ /dev/null @@ -1,63 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_duplicate_config_with_different_data(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.configs.add("test_config", "test_data", "/some/path") - with pytest.raises(Exception): - c1.configs.add("test_config", "test_data2", "/some/path") - - -def test_add_config_with_empty_target(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.configs.add("test_config", "test_data", "") - - -def test_add_duplicate_target(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.configs.add("test_config", "test_data", "/some/path") - with pytest.raises(Exception): - c1.configs.add("test_config2", "test_data2", "/some/path") - - -def test_add_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.configs.add("test_config", "$test_data", "/some/path") - output = render.render() - assert output["configs"]["test_config"]["content"] == "$$test_data" - assert output["services"]["test_container"]["configs"] == [{"source": "test_config", "target": "/some/path"}] - - -def test_add_config_with_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.configs.add("test_config", "test_data", "/some/path", "0777") - output = render.render() - assert output["configs"]["test_config"]["content"] == "test_data" - assert output["services"]["test_container"]["configs"] == [ - {"source": "test_config", "target": "/some/path", "mode": 511} - ] diff --git a/library/2.0.32/tests/test_container.py b/library/2.0.32/tests/test_container.py deleted file mode 100644 index 61a22a5df2..0000000000 --- a/library/2.0.32/tests/test_container.py +++ /dev/null @@ -1,324 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == "10s" - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 999] - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_tun_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_tun_device() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/dev/net/tun", - "target": "/dev/net/tun", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_snd_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_snd_device() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] - assert output["services"]["test_container"]["group_add"] == [29, 568] - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/library/2.0.32/tests/test_depends.py b/library/2.0.32/tests/test_depends.py deleted file mode 100644 index a1d8373927..0000000000 --- a/library/2.0.32/tests/test_depends.py +++ /dev/null @@ -1,54 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_dependency(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c2 = render.add_container("test_container2", "test_image") - c1.healthcheck.disable() - c2.healthcheck.disable() - c1.depends.add_dependency("test_container2", "service_started") - output = render.render() - assert output["services"]["test_container"]["depends_on"]["test_container2"] == {"condition": "service_started"} - - -def test_add_dependency_invalid_condition(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - render.add_container("test_container2", "test_image") - with pytest.raises(Exception): - c1.depends.add_dependency("test_container2", "invalid_condition") - - -def test_add_dependency_missing_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.depends.add_dependency("test_container2", "service_started") - - -def test_add_dependency_duplicate(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - render.add_container("test_container2", "test_image") - c1.depends.add_dependency("test_container2", "service_started") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.depends.add_dependency("test_container2", "service_started") diff --git a/library/2.0.32/tests/test_deps.py b/library/2.0.32/tests/test_deps.py deleted file mode 100644 index f9562ba4f2..0000000000 --- a/library/2.0.32/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/library/2.0.32/tests/test_device.py b/library/2.0.32/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/library/2.0.32/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/library/2.0.32/tests/test_dns.py b/library/2.0.32/tests/test_dns.py deleted file mode 100644 index fe6b21e34f..0000000000 --- a/library/2.0.32/tests/test_dns.py +++ /dev/null @@ -1,64 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_dns_opts(mock_values): - mock_values["network"] = {"dns_opts": ["attempts:3", "opt1", "opt2"]} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["dns_opt"] == ["attempts:3", "opt1", "opt2"] - - -def test_auto_add_dns_searches(mock_values): - mock_values["network"] = {"dns_searches": ["search1", "search2"]} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["dns_search"] == ["search1", "search2"] - - -def test_auto_add_dns_nameservers(mock_values): - mock_values["network"] = {"dns_nameservers": ["nameserver1", "nameserver2"]} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["dns"] == ["nameserver1", "nameserver2"] - - -def test_add_duplicate_dns_nameservers(mock_values): - mock_values["network"] = {"dns_nameservers": ["nameserver1", "nameserver1"]} - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_add_duplicate_dns_searches(mock_values): - mock_values["network"] = {"dns_searches": ["search1", "search1"]} - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_add_duplicate_dns_opts(mock_values): - mock_values["network"] = {"dns_opts": ["attempts:3", "attempts:5"]} - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") diff --git a/library/2.0.32/tests/test_environment.py b/library/2.0.32/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/library/2.0.32/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/library/2.0.32/tests/test_formatter.py b/library/2.0.32/tests/test_formatter.py deleted file mode 100644 index 843cf65d2e..0000000000 --- a/library/2.0.32/tests/test_formatter.py +++ /dev/null @@ -1,13 +0,0 @@ -from formatter import escape_dollar - - -def test_escape_dollar(): - cases = [ - {"input": "test", "expected": "test"}, - {"input": "$test", "expected": "$$test"}, - {"input": "$$test", "expected": "$$$$test"}, - {"input": "$$$test", "expected": "$$$$$$test"}, - {"input": "$test$", "expected": "$$test$$"}, - ] - for case in cases: - assert escape_dollar(case["input"]) == case["expected"] diff --git a/library/2.0.32/tests/test_functions.py b/library/2.0.32/tests/test_functions.py deleted file mode 100644 index 0ea3b57d18..0000000000 --- a/library/2.0.32/tests/test_functions.py +++ /dev/null @@ -1,88 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - mock_values["ix_volumes"] = {"test": "/mnt/test123"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, - {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, - { - "func": "get_host_path", - "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], - "expected": "/mnt/test", - }, - { - "func": "get_host_path", - "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], - "expected": "/mnt/test123", - }, - {"func": "or_default", "values": [None, 1], "expected": 1}, - {"func": "or_default", "values": [1, None], "expected": 1}, - {"func": "or_default", "values": [False, 1], "expected": 1}, - {"func": "or_default", "values": [True, 1], "expected": True}, - {"func": "temp_config", "values": [""], "expect_raise": True}, - { - "func": "temp_config", - "values": ["test"], - "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, - }, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/library/2.0.32/tests/test_healthcheck.py b/library/2.0.32/tests/test_healthcheck.py deleted file mode 100644 index fbd488ece4..0000000000 --- a/library/2.0.32/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/library/2.0.32/tests/test_labels.py b/library/2.0.32/tests/test_labels.py deleted file mode 100644 index ffa21eceac..0000000000 --- a/library/2.0.32/tests/test_labels.py +++ /dev/null @@ -1,88 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_disallowed_label(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.labels.add_label("com.docker.compose.service", "test_service") - - -def test_add_duplicate_label(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.labels.add_label("my.custom.label", "test_value") - with pytest.raises(Exception): - c1.labels.add_label("my.custom.label", "test_value1") - - -def test_add_label_on_non_existing_container(mock_values): - mock_values["labels"] = [ - { - "key": "my.custom.label1", - "value": "test_value1", - "containers": ["test_container", "test_container2"], - }, - ] - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.render() - - -def test_add_label(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.labels.add_label("my.custom.label1", "test_value1") - c1.labels.add_label("my.custom.label2", "test_value2") - output = render.render() - assert output["services"]["test_container"]["labels"] == { - "my.custom.label1": "test_value1", - "my.custom.label2": "test_value2", - } - - -def test_auto_add_labels(mock_values): - mock_values["labels"] = [ - { - "key": "my.custom.label1", - "value": "test_value1", - "containers": ["test_container", "test_container2"], - }, - { - "key": "my.custom.label2", - "value": "test_value2", - "containers": ["test_container"], - }, - ] - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c2 = render.add_container("test_container2", "test_image") - c1.healthcheck.disable() - c2.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["labels"] == { - "my.custom.label1": "test_value1", - "my.custom.label2": "test_value2", - } - assert output["services"]["test_container2"]["labels"] == { - "my.custom.label1": "test_value1", - } diff --git a/library/2.0.32/tests/test_notes.py b/library/2.0.32/tests/test_notes.py deleted file mode 100644 index 3613445385..0000000000 --- a/library/2.0.32/tests/test_notes.py +++ /dev/null @@ -1,213 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "ix_context": { - "app_metadata": { - "name": "test_app", - } - }, - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_notes(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert ( - output["x-notes"] - == """# Welcome to TrueNAS SCALE - -Thank you for installing test_app! - -## Documentation - -Documentation for test_app can be found at https://www.truenas.com/docs. - -## Bug reports - -If you find a bug in this app, please file an issue at -https://ixsystems.atlassian.net or https://github.com/truenas/apps - -## Feature requests or improvements - -If you find a feature request for this app, please file an issue at -https://ixsystems.atlassian.net or https://github.com/truenas/apps -""" - ) - - -def test_notes_with_warnings(mock_values): - render = Render(mock_values) - render.notes.add_warning("this is not properly configured. fix it now!") - render.notes.add_warning("that is not properly configured. fix it later!") - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert ( - output["x-notes"] - == """# Welcome to TrueNAS SCALE - -Thank you for installing test_app! - -## Warnings - -- this is not properly configured. fix it now! -- that is not properly configured. fix it later! - -## Documentation - -Documentation for test_app can be found at https://www.truenas.com/docs. - -## Bug reports - -If you find a bug in this app, please file an issue at -https://ixsystems.atlassian.net or https://github.com/truenas/apps - -## Feature requests or improvements - -If you find a feature request for this app, please file an issue at -https://ixsystems.atlassian.net or https://github.com/truenas/apps -""" - ) - - -def test_notes_with_deprecations(mock_values): - render = Render(mock_values) - render.notes.add_deprecation("this is will be removed later. fix it now!") - render.notes.add_deprecation("that is will be removed later. fix it later!") - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert ( - output["x-notes"] - == """# Welcome to TrueNAS SCALE - -Thank you for installing test_app! - -## Deprecations - -- this is will be removed later. fix it now! -- that is will be removed later. fix it later! - -## Documentation - -Documentation for test_app can be found at https://www.truenas.com/docs. - -## Bug reports - -If you find a bug in this app, please file an issue at -https://ixsystems.atlassian.net or https://github.com/truenas/apps - -## Feature requests or improvements - -If you find a feature request for this app, please file an issue at -https://ixsystems.atlassian.net or https://github.com/truenas/apps -""" - ) - - -def test_notes_with_body(mock_values): - render = Render(mock_values) - render.notes.set_body( - """## Additional info - -Some info -some other info. -""" - ) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert ( - output["x-notes"] - == """# Welcome to TrueNAS SCALE - -Thank you for installing test_app! - -## Additional info - -Some info -some other info. - -## Documentation - -Documentation for test_app can be found at https://www.truenas.com/docs. - -## Bug reports - -If you find a bug in this app, please file an issue at -https://ixsystems.atlassian.net or https://github.com/truenas/apps - -## Feature requests or improvements - -If you find a feature request for this app, please file an issue at -https://ixsystems.atlassian.net or https://github.com/truenas/apps -""" - ) - - -def test_notes_all(mock_values): - render = Render(mock_values) - render.notes.add_warning("this is not properly configured. fix it now!") - render.notes.add_warning("that is not properly configured. fix it later!") - render.notes.add_deprecation("this is will be removed later. fix it now!") - render.notes.add_deprecation("that is will be removed later. fix it later!") - render.notes.set_body( - """## Additional info - -Some info -some other info. -""" - ) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert ( - output["x-notes"] - == """# Welcome to TrueNAS SCALE - -Thank you for installing test_app! - -## Warnings - -- this is not properly configured. fix it now! -- that is not properly configured. fix it later! - -## Deprecations - -- this is will be removed later. fix it now! -- that is will be removed later. fix it later! - -## Additional info - -Some info -some other info. - -## Documentation - -Documentation for test_app can be found at https://www.truenas.com/docs. - -## Bug reports - -If you find a bug in this app, please file an issue at -https://ixsystems.atlassian.net or https://github.com/truenas/apps - -## Feature requests or improvements - -If you find a feature request for this app, please file an issue at -https://ixsystems.atlassian.net or https://github.com/truenas/apps -""" - ) diff --git a/library/2.0.32/tests/test_portal.py b/library/2.0.32/tests/test_portal.py deleted file mode 100644 index aebd9425c9..0000000000 --- a/library/2.0.32/tests/test_portal.py +++ /dev/null @@ -1,75 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_no_portals(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["x-portals"] == [] - - -def test_add_portal(mock_values): - render = Render(mock_values) - render.portals.add_portal({"scheme": "http", "path": "/", "port": 8080}) - render.portals.add_portal({"name": "Other Portal", "scheme": "https", "path": "/", "port": 8443}) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["x-portals"] == [ - {"name": "Other Portal", "scheme": "https", "host": "0.0.0.0", "port": 8443, "path": "/"}, - {"name": "Web UI", "scheme": "http", "host": "0.0.0.0", "port": 8080, "path": "/"}, - ] - - -def test_add_duplicate_portal(mock_values): - render = Render(mock_values) - render.portals.add_portal({"scheme": "http", "path": "/", "port": 8080}) - with pytest.raises(Exception): - render.portals.add_portal({"scheme": "http", "path": "/", "port": 8080}) - - -def test_add_duplicate_portal_with_explicit_name(mock_values): - render = Render(mock_values) - render.portals.add_portal({"name": "Some Portal", "scheme": "http", "path": "/", "port": 8080}) - with pytest.raises(Exception): - render.portals.add_portal({"name": "Some Portal", "scheme": "http", "path": "/", "port": 8080}) - - -def test_add_portal_with_invalid_scheme(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.portals.add_portal({"scheme": "invalid_scheme", "path": "/", "port": 8080}) - - -def test_add_portal_with_invalid_path(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.portals.add_portal({"scheme": "http", "path": "invalid_path", "port": 8080}) - - -def test_add_portal_with_invalid_path_double_slash(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.portals.add_portal({"scheme": "http", "path": "/some//path", "port": 8080}) - - -def test_add_portal_with_invalid_port(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.portals.add_portal({"scheme": "http", "path": "/", "port": -1}) diff --git a/library/2.0.32/tests/test_ports.py b/library/2.0.32/tests/test_ports.py deleted file mode 100644 index a4c923ca1d..0000000000 --- a/library/2.0.32/tests/test_ports.py +++ /dev/null @@ -1,110 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_ports(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.ports.add_port(8081, 8080) - c1.ports.add_port(8082, 8080, {"protocol": "udp"}) - output = render.render() - assert output["services"]["test_container"]["ports"] == [ - {"published": 8081, "target": 8080, "protocol": "tcp", "mode": "ingress", "host_ip": "0.0.0.0"}, - {"published": 8082, "target": 8080, "protocol": "udp", "mode": "ingress", "host_ip": "0.0.0.0"}, - ] - - -def test_add_duplicate_ports(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.ports.add_port(8081, 8080) - c1.ports.add_port(8081, 8080, {"protocol": "udp"}) # This should not raise - with pytest.raises(Exception): - c1.ports.add_port(8081, 8080) - - -def test_add_duplicate_ports_with_different_host_ip(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.ports.add_port(8081, 8080, {"host_ip": "192.168.1.10"}) - c1.ports.add_port(8081, 8080, {"host_ip": "192.168.1.11"}) - output = render.render() - assert output["services"]["test_container"]["ports"] == [ - {"published": 8081, "target": 8080, "protocol": "tcp", "mode": "ingress", "host_ip": "192.168.1.10"}, - {"published": 8081, "target": 8080, "protocol": "tcp", "mode": "ingress", "host_ip": "192.168.1.11"}, - ] - - -def test_add_duplicate_ports_to_specific_host_ip_binds_to_0_0_0_0(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.ports.add_port(8081, 8080, {"host_ip": "192.168.1.10"}) - with pytest.raises(Exception): - c1.ports.add_port(8081, 8080, {"host_ip": "0.0.0.0"}) - - -def test_add_duplicate_ports_to_0_0_0_0_binds_to_specific_host_ip(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.ports.add_port(8081, 8080, {"host_ip": "0.0.0.0"}) - with pytest.raises(Exception): - c1.ports.add_port(8081, 8080, {"host_ip": "192.168.1.10"}) - - -def test_add_ports_with_invalid_protocol(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.ports.add_port(8081, 8080, {"protocol": "invalid_protocol"}) - - -def test_add_ports_with_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.ports.add_port(8081, 8080, {"mode": "invalid_mode"}) - - -def test_add_ports_with_invalid_ip(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.ports.add_port(8081, 8080, {"host_ip": "invalid_ip"}) - - -def test_add_ports_with_invalid_host_port(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.ports.add_port(-1, 8080) - - -def test_add_ports_with_invalid_container_port(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.ports.add_port(8081, -1) diff --git a/library/2.0.32/tests/test_render.py b/library/2.0.32/tests/test_render.py deleted file mode 100644 index 60dc00679e..0000000000 --- a/library/2.0.32/tests/test_render.py +++ /dev/null @@ -1,37 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_values_cannot_be_modified(mock_values): - render = Render(mock_values) - render.values["test"] = "test" - with pytest.raises(Exception): - render.render() - - -def test_duplicate_containers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_no_containers(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.render() diff --git a/library/2.0.32/tests/test_resources.py b/library/2.0.32/tests/test_resources.py deleted file mode 100644 index cd83d164e5..0000000000 --- a/library/2.0.32/tests/test_resources.py +++ /dev/null @@ -1,140 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_automatically_add_cpu(mock_values): - mock_values["resources"] = {"limits": {"cpus": 1.0}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["deploy"]["resources"]["limits"]["cpus"] == "1.0" - - -def test_invalid_cpu(mock_values): - mock_values["resources"] = {"limits": {"cpus": "invalid"}} - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_automatically_add_memory(mock_values): - mock_values["resources"] = {"limits": {"memory": 1024}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["deploy"]["resources"]["limits"]["memory"] == "1024M" - - -def test_invalid_memory(mock_values): - mock_values["resources"] = {"limits": {"memory": "invalid"}} - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_automatically_add_gpus(mock_values): - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - devices = output["services"]["test_container"]["deploy"]["resources"]["reservations"]["devices"] - assert len(devices) == 1 - assert devices[0] == { - "capabilities": ["gpu"], - "driver": "nvidia", - "device_ids": ["uuid_0", "uuid_1"], - } - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_gpu_without_uuid(mock_values): - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_remove_cpus_and_memory_with_gpus(mock_values): - mock_values["resources"] = {"gpus": {"nvidia_gpu_selection": {"pci_slot_0": {"uuid": "uuid_1", "use_gpu": True}}}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.deploy.resources.remove_cpus_and_memory() - output = render.render() - assert "limits" not in output["services"]["test_container"]["deploy"]["resources"] - devices = output["services"]["test_container"]["deploy"]["resources"]["reservations"]["devices"] - assert len(devices) == 1 - assert devices[0] == { - "capabilities": ["gpu"], - "driver": "nvidia", - "device_ids": ["uuid_1"], - } - - -def test_remove_cpus_and_memory(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.deploy.resources.remove_cpus_and_memory() - output = render.render() - assert "deploy" not in output["services"]["test_container"] - - -def test_remove_devices(mock_values): - mock_values["resources"] = {"gpus": {"nvidia_gpu_selection": {"pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}}}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.deploy.resources.remove_devices() - output = render.render() - assert "reservations" not in output["services"]["test_container"]["deploy"]["resources"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_set_profile(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.deploy.resources.set_profile("low") - output = render.render() - assert output["services"]["test_container"]["deploy"]["resources"]["limits"]["cpus"] == "1" - assert output["services"]["test_container"]["deploy"]["resources"]["limits"]["memory"] == "512M" - - -def test_set_profile_invalid_profile(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.deploy.resources.set_profile("invalid_profile") diff --git a/library/2.0.32/tests/test_restart.py b/library/2.0.32/tests/test_restart.py deleted file mode 100644 index 06b2975590..0000000000 --- a/library/2.0.32/tests/test_restart.py +++ /dev/null @@ -1,57 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_invalid_restart_policy(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.restart.set_policy("invalid_policy") - - -def test_valid_restart_policy(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.restart.set_policy("on-failure") - output = render.render() - assert output["services"]["test_container"]["restart"] == "on-failure" - - -def test_valid_restart_policy_with_maximum_retry_count(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.restart.set_policy("on-failure", 10) - output = render.render() - assert output["services"]["test_container"]["restart"] == "on-failure:10" - - -def test_invalid_restart_policy_with_maximum_retry_count(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.restart.set_policy("on-failure", maximum_retry_count=-1) - - -def test_invalid_restart_policy_with_maximum_retry_count_and_policy(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.restart.set_policy("always", maximum_retry_count=10) diff --git a/library/2.0.32/tests/test_sysctls.py b/library/2.0.32/tests/test_sysctls.py deleted file mode 100644 index c9414044ea..0000000000 --- a/library/2.0.32/tests/test_sysctls.py +++ /dev/null @@ -1,62 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_sysctl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.sysctls.add("net.ipv4.ip_forward", 1) - c1.sysctls.add("fs.mqueue.msg_max", 100) - output = render.render() - assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} - - -def test_add_net_sysctl_with_host_network(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - c1.sysctls.add("net.ipv4.ip_forward", 1) - with pytest.raises(Exception): - render.render() - - -def test_add_duplicate_sysctl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.sysctls.add("net.ipv4.ip_forward", 1) - with pytest.raises(Exception): - c1.sysctls.add("net.ipv4.ip_forward", 0) - - -def test_add_empty_sysctl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.sysctls.add("", 1) - - -def test_add_sysctl_with_invalid_key(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.sysctls.add("invalid.sysctl", 1) - with pytest.raises(Exception): - render.render() diff --git a/library/2.0.32/tests/test_volumes.py b/library/2.0.32/tests/test_volumes.py deleted file mode 100644 index aef0d39481..0000000000 --- a/library/2.0.32/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage._add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/library/2.0.32/validations.py b/library/2.0.32/validations.py deleted file mode 100644 index 13f155dfdb..0000000000 --- a/library/2.0.32/validations.py +++ /dev/null @@ -1,227 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_sysctl_or_raise(sysctl: str, host_network: bool): - if not sysctl: - raise RenderError("Sysctl cannot be empty") - if host_network and sysctl.startswith("net."): - raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") - - valid_sysctls = [ - "kernel.msgmax", - "kernel.msgmnb", - "kernel.msgmni", - "kernel.sem", - "kernel.shmall", - "kernel.shmmax", - "kernel.shmmni", - "kernel.shm_rmid_forced", - ] - # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls - if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: - raise RenderError( - f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" - ) - return sysctl - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb", "/dev/snd"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/library/2.0.32/volume_mount.py b/library/2.0.32/volume_mount.py deleted file mode 100644 index aadd077750..0000000000 --- a/library/2.0.32/volume_mount.py +++ /dev/null @@ -1,92 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .formatter import merge_dicts_no_overwrite - from .volume_mount_types import BindMountType, VolumeMountType, TmpfsMountType - from .volume_sources import HostPathSource, IxVolumeSource, CifsSource, NfsSource, VolumeSource -except ImportError: - from error import RenderError - from formatter import merge_dicts_no_overwrite - from volume_mount_types import BindMountType, VolumeMountType, TmpfsMountType - from volume_sources import HostPathSource, IxVolumeSource, CifsSource, NfsSource, VolumeSource - - -class VolumeMount: - def __init__(self, render_instance: "Render", mount_path: str, config: "IxStorage"): - self._render_instance = render_instance - self.mount_path: str = mount_path - - storage_type: str = config.get("type", "") - if not storage_type: - raise RenderError("Expected [type] to be set for volume mounts.") - - match storage_type: - case "host_path": - spec_type = "bind" - mount_config = config.get("host_path_config") - if mount_config is None: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - mount_type_specific_definition = BindMountType(self._render_instance, mount_config).render() - source = HostPathSource(self._render_instance, mount_config).get() - case "ix_volume": - spec_type = "bind" - mount_config = config.get("ix_volume_config") - if mount_config is None: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - mount_type_specific_definition = BindMountType(self._render_instance, mount_config).render() - source = IxVolumeSource(self._render_instance, mount_config).get() - case "tmpfs": - spec_type = "tmpfs" - mount_config = config.get("tmpfs_config", {}) - mount_type_specific_definition = TmpfsMountType(self._render_instance, mount_config).render() - source = None - case "nfs": - spec_type = "volume" - mount_config = config.get("nfs_config") - if mount_config is None: - raise RenderError("Expected [nfs_config] to be set for [nfs] type.") - mount_type_specific_definition = VolumeMountType(self._render_instance, mount_config).render() - source = NfsSource(self._render_instance, mount_config).get() - case "cifs": - spec_type = "volume" - mount_config = config.get("cifs_config") - if mount_config is None: - raise RenderError("Expected [cifs_config] to be set for [cifs] type.") - mount_type_specific_definition = VolumeMountType(self._render_instance, mount_config).render() - source = CifsSource(self._render_instance, mount_config).get() - case "volume": - spec_type = "volume" - mount_config = config.get("volume_config") - if mount_config is None: - raise RenderError("Expected [volume_config] to be set for [volume] type.") - mount_type_specific_definition = VolumeMountType(self._render_instance, mount_config).render() - source = VolumeSource(self._render_instance, mount_config).get() - case "temporary": - spec_type = "volume" - mount_config = config.get("volume_config") - if mount_config is None: - raise RenderError("Expected [volume_config] to be set for [temporary] type.") - mount_type_specific_definition = VolumeMountType(self._render_instance, mount_config).render() - source = VolumeSource(self._render_instance, mount_config).get() - case "anonymous": - spec_type = "volume" - mount_config = config.get("volume_config") or {} - mount_type_specific_definition = VolumeMountType(self._render_instance, mount_config).render() - source = None - case _: - raise RenderError(f"Storage type [{storage_type}] is not supported for volume mounts.") - - common_spec = {"type": spec_type, "target": self.mount_path, "read_only": config.get("read_only", False)} - if source is not None: - common_spec["source"] = source - self._render_instance.volumes.add_volume(source, storage_type, mount_config) # type: ignore - - self.volume_mount_spec = merge_dicts_no_overwrite(common_spec, mount_type_specific_definition) - - def render(self) -> dict: - return self.volume_mount_spec diff --git a/library/2.0.32/volume_mount_types.py b/library/2.0.32/volume_mount_types.py deleted file mode 100644 index 00a0ec3a18..0000000000 --- a/library/2.0.32/volume_mount_types.py +++ /dev/null @@ -1,72 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorageTmpfsConfig, IxStorageVolumeConfig, IxStorageBindLikeConfigs - - -try: - from .error import RenderError - from .validations import valid_host_path_propagation, valid_octal_mode_or_raise -except ImportError: - from error import RenderError - from validations import valid_host_path_propagation, valid_octal_mode_or_raise - - -class TmpfsMountType: - def __init__(self, render_instance: "Render", config: "IxStorageTmpfsConfig"): - self._render_instance = render_instance - self.spec = {"tmpfs": {}} - size = config.get("size", None) - mode = config.get("mode", None) - - if size is not None: - if not isinstance(size, int): - raise RenderError(f"Expected [size] to be an integer for [tmpfs] type, got [{size}]") - if not size > 0: - raise RenderError(f"Expected [size] to be greater than 0 for [tmpfs] type, got [{size}]") - # Convert Mebibytes to Bytes - self.spec["tmpfs"]["size"] = size * 1024 * 1024 - - if mode is not None: - mode = valid_octal_mode_or_raise(mode) - self.spec["tmpfs"]["mode"] = int(mode, 8) - - if not self.spec["tmpfs"]: - self.spec.pop("tmpfs") - - def render(self) -> dict: - """Render the tmpfs mount specification.""" - return self.spec - - -class BindMountType: - def __init__(self, render_instance: "Render", config: "IxStorageBindLikeConfigs"): - self._render_instance = render_instance - self.spec: dict = {} - - propagation = valid_host_path_propagation(config.get("propagation", "rprivate")) - create_host_path = config.get("create_host_path", False) - - self.spec: dict = { - "bind": { - "create_host_path": create_host_path, - "propagation": propagation, - } - } - - def render(self) -> dict: - """Render the bind mount specification.""" - return self.spec - - -class VolumeMountType: - def __init__(self, render_instance: "Render", config: "IxStorageVolumeConfig"): - self._render_instance = render_instance - self.spec: dict = {} - - self.spec: dict = {"volume": {"nocopy": config.get("nocopy", False)}} - - def render(self) -> dict: - """Render the volume mount specification.""" - return self.spec diff --git a/library/2.0.32/volume_sources.py b/library/2.0.32/volume_sources.py deleted file mode 100644 index c33fe55ea1..0000000000 --- a/library/2.0.32/volume_sources.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorageHostPathConfig, IxStorageIxVolumeConfig, IxStorageVolumeConfig - -try: - from .error import RenderError - from .formatter import get_hashed_name_for_volume - from .validations import valid_fs_path_or_raise -except ImportError: - from error import RenderError - from formatter import get_hashed_name_for_volume - from validations import valid_fs_path_or_raise - - -class HostPathSource: - def __init__(self, render_instance: "Render", config: "IxStorageHostPathConfig"): - self._render_instance = render_instance - self.source: str = "" - - if not config: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - - path = "" - if config.get("acl_enable", False): - acl_path = config.get("acl", {}).get("path") - if not acl_path: - raise RenderError("Expected [host_path_config.acl.path] to be set for [host_path] type.") - path = valid_fs_path_or_raise(acl_path) - else: - path = valid_fs_path_or_raise(config.get("path", "")) - - self.source = path.rstrip("/") - - def get(self): - return self.source - - -class IxVolumeSource: - def __init__(self, render_instance: "Render", config: "IxStorageIxVolumeConfig"): - self._render_instance = render_instance - self.source: str = "" - - if not config: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - dataset_name = config.get("dataset_name") - if not dataset_name: - raise RenderError("Expected [ix_volume_config.dataset_name] to be set for [ix_volume] type.") - - ix_volumes = self._render_instance.values.get("ix_volumes", {}) - if dataset_name not in ix_volumes: - available = ", ".join(ix_volumes.keys()) - raise RenderError( - f"Expected the key [{dataset_name}] to be set in [ix_volumes] for [ix_volume] type. " - f"Available keys: [{available}]." - ) - - self.source = valid_fs_path_or_raise(ix_volumes[dataset_name].rstrip("/")) - - def get(self): - return self.source - - -class CifsSource: - def __init__(self, render_instance: "Render", config: dict): - self._render_instance = render_instance - self.source: str = "" - - if not config: - raise RenderError("Expected [cifs_config] to be set for [cifs] type.") - self.source = get_hashed_name_for_volume("cifs", config) - - def get(self): - return self.source - - -class NfsSource: - def __init__(self, render_instance: "Render", config: dict): - self._render_instance = render_instance - self.source: str = "" - - if not config: - raise RenderError("Expected [nfs_config] to be set for [nfs] type.") - self.source = get_hashed_name_for_volume("nfs", config) - - def get(self): - return self.source - - -class VolumeSource: - def __init__(self, render_instance: "Render", config: "IxStorageVolumeConfig"): - self._render_instance = render_instance - self.source: str = "" - - if not config: - raise RenderError("Expected [volume_config] to be set for [volume] type.") - - volume_name: str = config.get("volume_name", "") - if not volume_name: - raise RenderError("Expected [volume_config.volume_name] to be set for [volume] type.") - - self.source = volume_name - - def get(self): - return self.source diff --git a/library/2.0.32/volume_types.py b/library/2.0.32/volume_types.py deleted file mode 100644 index 4ccea08f83..0000000000 --- a/library/2.0.32/volume_types.py +++ /dev/null @@ -1,133 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorageNfsConfig, IxStorageCifsConfig, IxStorageVolumeConfig - - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_fs_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_fs_path_or_raise - - -class NfsVolume: - def __init__(self, render_instance: "Render", config: "IxStorageNfsConfig"): - self._render_instance = render_instance - - if not config: - raise RenderError("Expected [nfs_config] to be set for [nfs] type") - - required_keys = ["server", "path"] - for key in required_keys: - if not config.get(key): - raise RenderError(f"Expected [{key}] to be set for [nfs] type") - - opts = [f"addr={config['server']}"] - cfg_options = config.get("options") - if cfg_options: - if not isinstance(cfg_options, list): - raise RenderError("Expected [nfs_config.options] to be a list for [nfs] type") - - tracked_keys: set[str] = set() - disallowed_opts = ["addr"] - for opt in cfg_options: - if not isinstance(opt, str): - raise RenderError("Options for [nfs] type must be a list of strings.") - - key = opt.split("=")[0] - if key in tracked_keys: - raise RenderError(f"Option [{key}] already added for [nfs] type.") - if key in disallowed_opts: - raise RenderError(f"Option [{key}] is not allowed for [nfs] type.") - opts.append(opt) - tracked_keys.add(key) - - opts.sort() - - path = valid_fs_path_or_raise(config["path"].rstrip("/")) - self.volume_spec = { - "driver_opts": { - "type": "nfs", - "device": f":{path}", - "o": f"{','.join([escape_dollar(opt) for opt in opts])}", - }, - } - - def get(self): - return self.volume_spec - - -class CifsVolume: - def __init__(self, render_instance: "Render", config: "IxStorageCifsConfig"): - self._render_instance = render_instance - self.volume_spec: dict = {} - - if not config: - raise RenderError("Expected [cifs_config] to be set for [cifs] type") - - required_keys = ["server", "path", "username", "password"] - for key in required_keys: - if not config.get(key): - raise RenderError(f"Expected [{key}] to be set for [cifs] type") - - opts = [ - "noperm", - f"user={config['username']}", - f"password={config['password']}", - ] - - domain = config.get("domain") - if domain: - opts.append(f"domain={domain}") - - cfg_options = config.get("options") - if cfg_options: - if not isinstance(cfg_options, list): - raise RenderError("Expected [cifs_config.options] to be a list for [cifs] type") - - tracked_keys: set[str] = set() - disallowed_opts = ["user", "password", "domain", "noperm"] - for opt in cfg_options: - if not isinstance(opt, str): - raise RenderError("Options for [cifs] type must be a list of strings.") - - key = opt.split("=")[0] - if key in tracked_keys: - raise RenderError(f"Option [{key}] already added for [cifs] type.") - if key in disallowed_opts: - raise RenderError(f"Option [{key}] is not allowed for [cifs] type.") - for disallowed in disallowed_opts: - if key == disallowed: - raise RenderError(f"Option [{key}] is not allowed for [cifs] type.") - opts.append(opt) - tracked_keys.add(key) - opts.sort() - - server = config["server"].lstrip("/") - path = config["path"].strip("/") - path = valid_fs_path_or_raise("/" + path).lstrip("/") - - self.volume_spec = { - "driver_opts": { - "type": "cifs", - "device": f"//{server}/{path}", - "o": f"{','.join([escape_dollar(opt) for opt in opts])}", - }, - } - - def get(self): - return self.volume_spec - - -class DockerVolume: - def __init__(self, render_instance: "Render", config: "IxStorageVolumeConfig"): - self._render_instance = render_instance - self.volume_spec: dict = {} - - def get(self): - return self.volume_spec diff --git a/library/2.0.32/volumes.py b/library/2.0.32/volumes.py deleted file mode 100644 index e6925a402f..0000000000 --- a/library/2.0.32/volumes.py +++ /dev/null @@ -1,66 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - - -try: - from .error import RenderError - from .storage import IxStorageVolumeLikeConfigs - from .volume_types import NfsVolume, CifsVolume, DockerVolume -except ImportError: - from error import RenderError - from storage import IxStorageVolumeLikeConfigs - from volume_types import NfsVolume, CifsVolume, DockerVolume - - -class Volumes: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volumes: dict[str, Volume] = {} - - def add_volume( - self, - source: str, - storage_type: str, - config: "IxStorageVolumeLikeConfigs", - ): - # This method can be called many times from the volume mounts - # Only add the volume if it is not already added, but dont raise an error - if source == "": - raise RenderError(f"Volume source [{source}] cannot be empty") - - if source in self._volumes: - return - - self._volumes[source] = Volume(self._render_instance, storage_type, config) - - def has_volumes(self) -> bool: - return bool(self._volumes) - - def render(self): - return {name: v.render() for name, v in sorted(self._volumes.items()) if v.render() is not None} - - -class Volume: - def __init__( - self, - render_instance: "Render", - storage_type: str, - config: "IxStorageVolumeLikeConfigs", - ): - self._render_instance = render_instance - self.volume_spec: dict | None = {} - - match storage_type: - case "nfs": - self.volume_spec = NfsVolume(self._render_instance, config).get() # type: ignore - case "cifs": - self.volume_spec = CifsVolume(self._render_instance, config).get() # type: ignore - case "volume" | "temporary": - self.volume_spec = DockerVolume(self._render_instance, config).get() # type: ignore - case _: - self.volume_spec = None - - def render(self): - return self.volume_spec From c97e2871434b05b63c5ab1f722c5492704db8bc4 Mon Sep 17 00:00:00 2001 From: Stavros Kois Date: Wed, 4 Dec 2024 15:01:55 +0200 Subject: [PATCH 17/19] regen hash --- library/hashes.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/hashes.yaml b/library/hashes.yaml index e838332e5f..a470c76c17 100644 --- a/library/hashes.yaml +++ b/library/hashes.yaml @@ -1,3 +1,3 @@ 0.0.1: f074617a82a86d2a6cc78a4c8a4296fc9d168e456f12713e50c696557b302133 1.1.7: d05e43e25b7dc1736be6cc1efa4b9255368aa346e3e7a4350a38440f29b73186 -2.0.33: b30cf75dc79622680a2c740be76709b35ea33cb67852a4bbc1305cbe7c1cf5c3 +2.0.33: 167a3532cd10bbb030a484b3b2a84248fad6d59e9ad0338ad0bb6c543dfca949 From 00abcbb1f879b21633c32c7f2d9d7ecd94a9ee0f Mon Sep 17 00:00:00 2001 From: Stavros Kois Date: Wed, 4 Dec 2024 20:15:09 +0200 Subject: [PATCH 18/19] ix-app: use v2 --- ix-dev/stable/ix-app/app.yaml | 6 +- ix-dev/stable/ix-app/questions.yaml | 37 + .../ix-app/templates/docker-compose.yaml | 163 +++-- .../library/base_v1_1_7/environment.py | 98 --- .../library/base_v1_1_7/healthchecks.py | 120 ---- .../templates/library/base_v1_1_7/mariadb.py | 72 -- .../templates/library/base_v1_1_7/metadata.py | 71 -- .../templates/library/base_v1_1_7/network.py | 21 - .../library/base_v1_1_7/permissions.py | 139 ---- .../templates/library/base_v1_1_7/ports.py | 42 -- .../templates/library/base_v1_1_7/postgres.py | 77 -- .../templates/library/base_v1_1_7/redis.py | 49 -- .../library/base_v1_1_7/resources.py | 101 --- .../templates/library/base_v1_1_7/security.py | 34 - .../templates/library/base_v1_1_7/storage.py | 370 ---------- .../templates/library/base_v1_1_7/utils.py | 124 ---- .../{base_v1_1_7 => base_v2_0_34}/__init__.py | 0 .../library/base_v2_0_34}/configs.py | 0 .../library/base_v2_0_34}/container.py | 30 +- .../library/base_v2_0_34}/depends.py | 0 .../templates/library/base_v2_0_34}/deploy.py | 0 .../templates/library/base_v2_0_34}/deps.py | 0 .../templates/library/base_v2_0_34}/device.py | 0 .../library/base_v2_0_34}/devices.py | 0 .../templates/library/base_v2_0_34}/dns.py | 0 .../library/base_v2_0_34}/environment.py | 5 +- .../templates/library/base_v2_0_34}/error.py | 0 .../library/base_v2_0_34}/formatter.py | 0 .../library/base_v2_0_34}/functions.py | 0 .../library/base_v2_0_34}/healthcheck.py | 10 + .../templates/library/base_v2_0_34}/labels.py | 0 .../templates/library/base_v2_0_34}/notes.py | 0 .../templates/library/base_v2_0_34}/portal.py | 0 .../library/base_v2_0_34}/portals.py | 0 .../templates/library/base_v2_0_34}/ports.py | 0 .../templates/library/base_v2_0_34}/render.py | 0 .../library/base_v2_0_34}/resources.py | 0 .../library/base_v2_0_34}/restart.py | 0 .../library/base_v2_0_34}/storage.py | 0 .../library/base_v2_0_34}/sysctls.py | 0 .../library/base_v2_0_34/tests}/__init__.py | 0 .../base_v2_0_34}/tests/test_build_image.py | 0 .../base_v2_0_34}/tests/test_configs.py | 0 .../base_v2_0_34}/tests/test_container.py | 36 + .../base_v2_0_34}/tests/test_depends.py | 0 .../library/base_v2_0_34}/tests/test_deps.py | 0 .../base_v2_0_34}/tests/test_device.py | 0 .../library/base_v2_0_34}/tests/test_dns.py | 0 .../base_v2_0_34}/tests/test_environment.py | 12 + .../base_v2_0_34}/tests/test_formatter.py | 0 .../base_v2_0_34}/tests/test_functions.py | 0 .../base_v2_0_34}/tests/test_healthcheck.py | 8 + .../base_v2_0_34}/tests/test_labels.py | 0 .../library/base_v2_0_34}/tests/test_notes.py | 0 .../base_v2_0_34}/tests/test_portal.py | 0 .../library/base_v2_0_34}/tests/test_ports.py | 0 .../base_v2_0_34}/tests/test_render.py | 0 .../base_v2_0_34}/tests/test_resources.py | 0 .../base_v2_0_34}/tests/test_restart.py | 0 .../base_v2_0_34}/tests/test_sysctls.py | 0 .../base_v2_0_34}/tests/test_volumes.py | 0 .../library/base_v2_0_34}/validations.py | 7 + .../library/base_v2_0_34}/volume_mount.py | 0 .../base_v2_0_34}/volume_mount_types.py | 0 .../library/base_v2_0_34}/volume_sources.py | 0 .../library/base_v2_0_34}/volume_types.py | 0 .../library/base_v2_0_34}/volumes.py | 0 .../templates/test_values/basic-values.yaml | 36 +- library/{2.0.33/tests => 2.0.34}/__init__.py | 0 library/2.0.34/configs.py | 86 +++ library/2.0.34/container.py | 339 +++++++++ library/2.0.34/depends.py | 34 + library/2.0.34/deploy.py | 24 + library/2.0.34/deps.py | 454 ++++++++++++ library/2.0.34/device.py | 31 + library/2.0.34/devices.py | 68 ++ library/2.0.34/dns.py | 79 +++ library/2.0.34/environment.py | 112 +++ library/2.0.34/error.py | 4 + library/2.0.34/formatter.py | 26 + library/2.0.34/functions.py | 149 ++++ library/2.0.34/healthcheck.py | 203 ++++++ library/2.0.34/labels.py | 37 + library/2.0.34/notes.py | 70 ++ library/2.0.34/portal.py | 22 + library/2.0.34/portals.py | 28 + library/2.0.34/ports.py | 68 ++ library/2.0.34/render.py | 89 +++ library/2.0.34/resources.py | 115 +++ library/2.0.34/restart.py | 25 + library/2.0.34/storage.py | 116 +++ library/2.0.34/sysctls.py | 38 + library/2.0.34/tests/__init__.py | 0 library/2.0.34/tests/test_build_image.py | 49 ++ library/2.0.34/tests/test_configs.py | 63 ++ library/2.0.34/tests/test_container.py | 360 ++++++++++ library/2.0.34/tests/test_depends.py | 54 ++ library/2.0.34/tests/test_deps.py | 380 ++++++++++ library/2.0.34/tests/test_device.py | 131 ++++ library/2.0.34/tests/test_dns.py | 64 ++ library/2.0.34/tests/test_environment.py | 196 ++++++ library/2.0.34/tests/test_formatter.py | 13 + library/2.0.34/tests/test_functions.py | 88 +++ library/2.0.34/tests/test_healthcheck.py | 195 +++++ library/2.0.34/tests/test_labels.py | 88 +++ library/2.0.34/tests/test_notes.py | 213 ++++++ library/2.0.34/tests/test_portal.py | 75 ++ library/2.0.34/tests/test_ports.py | 110 +++ library/2.0.34/tests/test_render.py | 37 + library/2.0.34/tests/test_resources.py | 140 ++++ library/2.0.34/tests/test_restart.py | 57 ++ library/2.0.34/tests/test_sysctls.py | 62 ++ library/2.0.34/tests/test_volumes.py | 666 ++++++++++++++++++ library/2.0.34/validations.py | 234 ++++++ library/2.0.34/volume_mount.py | 92 +++ library/2.0.34/volume_mount_types.py | 72 ++ library/2.0.34/volume_sources.py | 106 +++ library/2.0.34/volume_types.py | 133 ++++ library/2.0.34/volumes.py | 66 ++ library/hashes.yaml | 2 +- 120 files changed, 6224 insertions(+), 1407 deletions(-) delete mode 100644 ix-dev/stable/ix-app/templates/library/base_v1_1_7/environment.py delete mode 100644 ix-dev/stable/ix-app/templates/library/base_v1_1_7/healthchecks.py delete mode 100644 ix-dev/stable/ix-app/templates/library/base_v1_1_7/mariadb.py delete mode 100644 ix-dev/stable/ix-app/templates/library/base_v1_1_7/metadata.py delete mode 100644 ix-dev/stable/ix-app/templates/library/base_v1_1_7/network.py delete mode 100644 ix-dev/stable/ix-app/templates/library/base_v1_1_7/permissions.py delete mode 100644 ix-dev/stable/ix-app/templates/library/base_v1_1_7/ports.py delete mode 100644 ix-dev/stable/ix-app/templates/library/base_v1_1_7/postgres.py delete mode 100644 ix-dev/stable/ix-app/templates/library/base_v1_1_7/redis.py delete mode 100644 ix-dev/stable/ix-app/templates/library/base_v1_1_7/resources.py delete mode 100644 ix-dev/stable/ix-app/templates/library/base_v1_1_7/security.py delete mode 100644 ix-dev/stable/ix-app/templates/library/base_v1_1_7/storage.py delete mode 100644 ix-dev/stable/ix-app/templates/library/base_v1_1_7/utils.py rename ix-dev/stable/ix-app/templates/library/{base_v1_1_7 => base_v2_0_34}/__init__.py (100%) rename {library/2.0.33 => ix-dev/stable/ix-app/templates/library/base_v2_0_34}/configs.py (100%) rename {library/2.0.33 => ix-dev/stable/ix-app/templates/library/base_v2_0_34}/container.py (93%) rename {library/2.0.33 => ix-dev/stable/ix-app/templates/library/base_v2_0_34}/depends.py (100%) rename {library/2.0.33 => ix-dev/stable/ix-app/templates/library/base_v2_0_34}/deploy.py (100%) rename {library/2.0.33 => ix-dev/stable/ix-app/templates/library/base_v2_0_34}/deps.py (100%) rename {library/2.0.33 => ix-dev/stable/ix-app/templates/library/base_v2_0_34}/device.py (100%) rename {library/2.0.33 => ix-dev/stable/ix-app/templates/library/base_v2_0_34}/devices.py (100%) rename {library/2.0.33 => ix-dev/stable/ix-app/templates/library/base_v2_0_34}/dns.py (100%) rename {library/2.0.33 => ix-dev/stable/ix-app/templates/library/base_v2_0_34}/environment.py (95%) rename {library/2.0.33 => ix-dev/stable/ix-app/templates/library/base_v2_0_34}/error.py (100%) rename {library/2.0.33 => ix-dev/stable/ix-app/templates/library/base_v2_0_34}/formatter.py (100%) rename {library/2.0.33 => ix-dev/stable/ix-app/templates/library/base_v2_0_34}/functions.py (100%) rename {library/2.0.33 => ix-dev/stable/ix-app/templates/library/base_v2_0_34}/healthcheck.py (95%) rename {library/2.0.33 => ix-dev/stable/ix-app/templates/library/base_v2_0_34}/labels.py (100%) rename {library/2.0.33 => ix-dev/stable/ix-app/templates/library/base_v2_0_34}/notes.py (100%) rename {library/2.0.33 => ix-dev/stable/ix-app/templates/library/base_v2_0_34}/portal.py (100%) rename {library/2.0.33 => ix-dev/stable/ix-app/templates/library/base_v2_0_34}/portals.py (100%) rename {library/2.0.33 => ix-dev/stable/ix-app/templates/library/base_v2_0_34}/ports.py (100%) rename {library/2.0.33 => ix-dev/stable/ix-app/templates/library/base_v2_0_34}/render.py (100%) rename {library/2.0.33 => ix-dev/stable/ix-app/templates/library/base_v2_0_34}/resources.py (100%) rename {library/2.0.33 => ix-dev/stable/ix-app/templates/library/base_v2_0_34}/restart.py (100%) rename {library/2.0.33 => ix-dev/stable/ix-app/templates/library/base_v2_0_34}/storage.py (100%) rename {library/2.0.33 => ix-dev/stable/ix-app/templates/library/base_v2_0_34}/sysctls.py (100%) rename {library/2.0.33 => ix-dev/stable/ix-app/templates/library/base_v2_0_34/tests}/__init__.py (100%) rename {library/2.0.33 => ix-dev/stable/ix-app/templates/library/base_v2_0_34}/tests/test_build_image.py (100%) rename {library/2.0.33 => ix-dev/stable/ix-app/templates/library/base_v2_0_34}/tests/test_configs.py (100%) rename {library/2.0.33 => ix-dev/stable/ix-app/templates/library/base_v2_0_34}/tests/test_container.py (89%) rename {library/2.0.33 => ix-dev/stable/ix-app/templates/library/base_v2_0_34}/tests/test_depends.py (100%) rename {library/2.0.33 => ix-dev/stable/ix-app/templates/library/base_v2_0_34}/tests/test_deps.py (100%) rename {library/2.0.33 => ix-dev/stable/ix-app/templates/library/base_v2_0_34}/tests/test_device.py (100%) rename {library/2.0.33 => ix-dev/stable/ix-app/templates/library/base_v2_0_34}/tests/test_dns.py (100%) rename {library/2.0.33 => ix-dev/stable/ix-app/templates/library/base_v2_0_34}/tests/test_environment.py (93%) rename {library/2.0.33 => ix-dev/stable/ix-app/templates/library/base_v2_0_34}/tests/test_formatter.py (100%) rename {library/2.0.33 => ix-dev/stable/ix-app/templates/library/base_v2_0_34}/tests/test_functions.py (100%) rename {library/2.0.33 => ix-dev/stable/ix-app/templates/library/base_v2_0_34}/tests/test_healthcheck.py (95%) rename {library/2.0.33 => ix-dev/stable/ix-app/templates/library/base_v2_0_34}/tests/test_labels.py (100%) rename {library/2.0.33 => ix-dev/stable/ix-app/templates/library/base_v2_0_34}/tests/test_notes.py (100%) rename {library/2.0.33 => ix-dev/stable/ix-app/templates/library/base_v2_0_34}/tests/test_portal.py (100%) rename {library/2.0.33 => ix-dev/stable/ix-app/templates/library/base_v2_0_34}/tests/test_ports.py (100%) rename {library/2.0.33 => ix-dev/stable/ix-app/templates/library/base_v2_0_34}/tests/test_render.py (100%) rename {library/2.0.33 => ix-dev/stable/ix-app/templates/library/base_v2_0_34}/tests/test_resources.py (100%) rename {library/2.0.33 => ix-dev/stable/ix-app/templates/library/base_v2_0_34}/tests/test_restart.py (100%) rename {library/2.0.33 => ix-dev/stable/ix-app/templates/library/base_v2_0_34}/tests/test_sysctls.py (100%) rename {library/2.0.33 => ix-dev/stable/ix-app/templates/library/base_v2_0_34}/tests/test_volumes.py (100%) rename {library/2.0.33 => ix-dev/stable/ix-app/templates/library/base_v2_0_34}/validations.py (95%) rename {library/2.0.33 => ix-dev/stable/ix-app/templates/library/base_v2_0_34}/volume_mount.py (100%) rename {library/2.0.33 => ix-dev/stable/ix-app/templates/library/base_v2_0_34}/volume_mount_types.py (100%) rename {library/2.0.33 => ix-dev/stable/ix-app/templates/library/base_v2_0_34}/volume_sources.py (100%) rename {library/2.0.33 => ix-dev/stable/ix-app/templates/library/base_v2_0_34}/volume_types.py (100%) rename {library/2.0.33 => ix-dev/stable/ix-app/templates/library/base_v2_0_34}/volumes.py (100%) rename library/{2.0.33/tests => 2.0.34}/__init__.py (100%) create mode 100644 library/2.0.34/configs.py create mode 100644 library/2.0.34/container.py create mode 100644 library/2.0.34/depends.py create mode 100644 library/2.0.34/deploy.py create mode 100644 library/2.0.34/deps.py create mode 100644 library/2.0.34/device.py create mode 100644 library/2.0.34/devices.py create mode 100644 library/2.0.34/dns.py create mode 100644 library/2.0.34/environment.py create mode 100644 library/2.0.34/error.py create mode 100644 library/2.0.34/formatter.py create mode 100644 library/2.0.34/functions.py create mode 100644 library/2.0.34/healthcheck.py create mode 100644 library/2.0.34/labels.py create mode 100644 library/2.0.34/notes.py create mode 100644 library/2.0.34/portal.py create mode 100644 library/2.0.34/portals.py create mode 100644 library/2.0.34/ports.py create mode 100644 library/2.0.34/render.py create mode 100644 library/2.0.34/resources.py create mode 100644 library/2.0.34/restart.py create mode 100644 library/2.0.34/storage.py create mode 100644 library/2.0.34/sysctls.py create mode 100644 library/2.0.34/tests/__init__.py create mode 100644 library/2.0.34/tests/test_build_image.py create mode 100644 library/2.0.34/tests/test_configs.py create mode 100644 library/2.0.34/tests/test_container.py create mode 100644 library/2.0.34/tests/test_depends.py create mode 100644 library/2.0.34/tests/test_deps.py create mode 100644 library/2.0.34/tests/test_device.py create mode 100644 library/2.0.34/tests/test_dns.py create mode 100644 library/2.0.34/tests/test_environment.py create mode 100644 library/2.0.34/tests/test_formatter.py create mode 100644 library/2.0.34/tests/test_functions.py create mode 100644 library/2.0.34/tests/test_healthcheck.py create mode 100644 library/2.0.34/tests/test_labels.py create mode 100644 library/2.0.34/tests/test_notes.py create mode 100644 library/2.0.34/tests/test_portal.py create mode 100644 library/2.0.34/tests/test_ports.py create mode 100644 library/2.0.34/tests/test_render.py create mode 100644 library/2.0.34/tests/test_resources.py create mode 100644 library/2.0.34/tests/test_restart.py create mode 100644 library/2.0.34/tests/test_sysctls.py create mode 100644 library/2.0.34/tests/test_volumes.py create mode 100644 library/2.0.34/validations.py create mode 100644 library/2.0.34/volume_mount.py create mode 100644 library/2.0.34/volume_mount_types.py create mode 100644 library/2.0.34/volume_sources.py create mode 100644 library/2.0.34/volume_types.py create mode 100644 library/2.0.34/volumes.py diff --git a/ix-dev/stable/ix-app/app.yaml b/ix-dev/stable/ix-app/app.yaml index 207d69fff8..5441a4e9a4 100644 --- a/ix-dev/stable/ix-app/app.yaml +++ b/ix-dev/stable/ix-app/app.yaml @@ -7,8 +7,8 @@ home: https://www.truenas.com/ host_mounts: [] icon: https://media.sys.truenas.net/apps/ix-chart/icons/icon.webp keywords: [] -lib_version: 1.1.7 -lib_version_hash: d05e43e25b7dc1736be6cc1efa4b9255368aa346e3e7a4350a38440f29b73186 +lib_version: 2.0.34 +lib_version_hash: 54c507c0a47e2a10aee421ec5a67c04f73692bd865d4d5057f57711e322a7951 maintainers: - email: dev@ixsystems.com name: truenas @@ -19,4 +19,4 @@ screenshots: [] sources: [] title: iX App train: stable -version: 1.0.14 +version: 1.1.0 diff --git a/ix-dev/stable/ix-app/questions.yaml b/ix-dev/stable/ix-app/questions.yaml index 5382d664aa..351c9f8eaf 100644 --- a/ix-dev/stable/ix-app/questions.yaml +++ b/ix-dev/stable/ix-app/questions.yaml @@ -11,6 +11,8 @@ groups: description: Configure Portals - name: Storage Configuration description: Configure Storage for the container + - name: Labels Configuration + description: Configure Labels for the container - name: Resources Configuration description: Configure Resources for the container @@ -135,6 +137,17 @@ questions: description: On Failure - Restarts the container if the exit code indicates an error. - value: always description: Always - Restarts the container until its removal. + - variable: max_retry_count + label: Maximum Retry Count + group: Container Configuration + description: | + Maximum number of retries allowed for a container to exit with a code indicating an error.
+ Setting this to zero, will keep restarting the container if it exits with a code indicating an error. + schema: + type: int + required: true + default: 0 + show_if: [["restart_policy", "on-failure"]] - variable: disable_builtin_healthcheck label: Disable Builtin Healthcheck @@ -530,6 +543,30 @@ questions: type: int default: 500 required: true + + - variable: labels + label: "" + group: Labels Configuration + schema: + type: list + default: [] + items: + - variable: label + label: Label + schema: + type: dict + attrs: + - variable: key + label: Key + schema: + type: string + required: true + - variable: value + label: Value + schema: + type: string + required: true + - variable: resources label: "" group: Resources Configuration diff --git a/ix-dev/stable/ix-app/templates/docker-compose.yaml b/ix-dev/stable/ix-app/templates/docker-compose.yaml index 94adeb6434..58622b3b24 100644 --- a/ix-dev/stable/ix-app/templates/docker-compose.yaml +++ b/ix-dev/stable/ix-app/templates/docker-compose.yaml @@ -1,73 +1,90 @@ -services: - {{ values.ix_context.app_name }}: - {# Image Configuration #} - image: {{ "%s:%s" | format(values.image.repository, (values.image.tag or "latest")) }} - - {# Container Configuration #} - restart: {{ values.restart_policy }} - tty: {{ values.tty }} - stdin_open: {{ values.stdin }} - - {% if values.entrypoint %} - entrypoint: {{ values.entrypoint | tojson }} - {% endif %} - - {% if values.command %} - command: {{ values.command | tojson }} - {% endif %} - - {% if values.disable_builtin_healthcheck %} - healthcheck: {"disable": true} - {% endif %} - - environment: {{ ix_lib.base.environment.envs(app={}, user=values.envs, values={"TZ": values.TZ}) | tojson }} - {# Network Configuration #} - {% if values.dns_config.nameservers %} - dns: {{ values.dns_config.nameservers | tojson }} - {% endif %} - - {% if values.dns_config.searches %} - dns_search: {{ values.dns_config.searches | tojson }} - {% endif %} - - {% if values.dns_config.options %} - dns_opt: {{ ix_lib.base.network.dns_opts(values.dns_config.options) | tojson }} - {% endif %} - - {% if values.host_network %} - network_mode: host - {% endif %} - - {% if not values.host_network and values.ports %} - ports: - {% for port in values.ports %} - - {{ ix_lib.base.ports.get_port(port=port) | tojson }} - {% endfor %} - {% endif %} - - {# Security Context Configuration #} - privileged: {{ values.privileged }} - {% set caps = ix_lib.base.security.get_caps(add=values.capabilities.add, drop=[]) %} - {% if caps.add %} - cap_add: {{ caps.add | tojson }} - {% endif %} - - {% if values.run_as_custom_user %} - user: {{ "%d:%d" | format(values.run_as.user, values.run_as.group) }} - {% endif %} - - {# Storage Configuration #} - {% if values.storage %} - volumes: - {% for store in values.storage %} - - {{ ix_lib.base.storage.storage_item(data=store, values=values).vol_mount | tojson }} - {% endfor %} - {% endif %} - - {# Resources Configuration #} - deploy: - resources: {{ ix_lib.base.resources.resources(values.resources, not values.resources.enable_resource_limits) | tojson }} - devices: {{ ix_lib.base.resources.get_devices(values.resources) | tojson }} - -x-portals: {{ ix_lib.base.metadata.get_portals(values.portals) | tojson }} -x-notes: {{ ix_lib.base.metadata.get_notes("iX App") | tojson }} +{# Adjust values to library will pick some things automatically #} +{% do values.update({ + "skip_generic_variables": true, + "images": { + "image": { + "repository": values.image.repository, + "tag": values.image.tag or "latest", + } + }, + "network": { + "dns_opts": values.dns_config.get("options", []), + "dns_searches": values.dns_config.get("searches", []), + "dns_nameservers": values.dns_config.get("nameservers", []), + } +}) %} + +{% for label in values.labels %} + {% do label.update({"containers": [values.ix_context.app_name]}) %} +{% endfor %} + +{# Any manipulation to values should be done before this point #} + +{# Template starts here #} +{% set tpl = ix_lib.base.render.Render(values) %} + +{# Image Configuration #} +{% set c1 = tpl.add_container(values.ix_context.app_name, "image") %} +{% do c1.set_pull_policy(values.image.pull_policy) %} + +{# Container Configuration #} +{% if values.restart_policy == "on-failure" %} + {% do c1.restart.set_policy(values.restart_policy, values.max_retry_count) %} +{% else %} + {% do c1.restart.set_policy(values.restart_policy) %} +{% endif %} + +{% do c1.set_tty(values.tty) %} +{% do c1.set_stdin(values.stdin) %} + +{% if values.entrypoint %} + {% do c1.set_entrypoint(values.entrypoint) %} +{% endif %} + +{% if values.command %} + {% do c1.set_command(values.command) %} +{% endif %} + +{% if values.disable_builtin_healthcheck %} + {% do c1.healthcheck.disable() %} +{% else %} + {% do c1.healthcheck.use_built_in() %} +{% endif %} + +{% do c1.environment.add_env("TZ", values.TZ) %} +{% do c1.environment.add_user_envs(values.envs) %} + +{# Network Configuration #} +{% if values.host_network %} + {% do c1.set_network_mode("host") %} +{% else %} + {% if values.ports %} + {% for port in values.ports %} + {% do c1.ports.add_port(port.published, port.target, {"protocol": port.protocol}) %} + {% endfor %} + {% endif %} +{% endif %} + +{# Security Context Configuration #} +{% do c1.set_privileged(values.privileged) %} +{% do c1.clear_caps() %} +{% do c1.remove_security_opt("no-new-privileges") %} + +{% do c1.add_caps(values.capabilities.add) %} +{% if values.run_as_custom_user %} + {% do c1.set_user(values.run_as.user, values.run_as.group) %} +{% endif %} + +{% for store in values.storage %} + {% do c1.add_storage(store.mount_path, store) %} +{% endfor %} + +{% if not values.resources.enable_resource_limits %} + {% do c1.deploy.resources.remove_cpus_and_memory() %} +{% endif %} + +{% for portal in values.portals %} + {% do tpl.portals.add_portal(portal) %} +{% endfor %} + +{{ tpl.render() | tojson }} diff --git a/ix-dev/stable/ix-app/templates/library/base_v1_1_7/environment.py b/ix-dev/stable/ix-app/templates/library/base_v1_1_7/environment.py deleted file mode 100644 index be5c8b7347..0000000000 --- a/ix-dev/stable/ix-app/templates/library/base_v1_1_7/environment.py +++ /dev/null @@ -1,98 +0,0 @@ -from . import utils -from .resources import get_nvidia_gpus_reservations - - -def envs(app: dict | None = None, user: list | None = None, values: dict | None = None): - app = app or {} - user = user or [] - values = values or {} - result = {} - - if not values: - utils.throw_error("Values cannot be empty in environment.py") - - if not isinstance(user, list): - utils.throw_error( - f"Unsupported type for user environment variables [{type(user)}]" - ) - - # Always set TZ - result.update({"TZ": values.get("TZ", "Etc/UTC")}) - - # Update envs with nvidia variables - if values.get("resources", {}).get("gpus", {}): - result.update(get_nvidia_env(values.get("resources", {}).get("gpus", {}))) - - # Update envs with run_as variables - if values.get("run_as"): - result.update(get_run_as_envs(values.get("run_as", {}))) - - # Make sure we don't manually set any of the above - for item in app.items(): - if not item[0]: - utils.throw_error("Environment variable name cannot be empty.") - if item[0] in result: - utils.throw_error( - f"Environment variable [{item[0]}] is already defined automatically from the library." - ) - result[item[0]] = item[1] - - for item in user: - if not item.get("name"): - utils.throw_error("Environment variable name cannot be empty.") - if item.get("name") in result: - utils.throw_error( - f"Environment variable [{item['name']}] is already defined from the application developer." - ) - result[item["name"]] = item.get("value") - - for k, v in result.items(): - val = str(v) - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - val = val.lower() - result[k] = utils.escape_dollar(val) - - return result - - -# Sets some common variables that most applications use -def get_run_as_envs(run_as: dict) -> dict: - result = {} - user = run_as.get("user") - group = run_as.get("group") - if user: - result.update( - { - "PUID": user, - "UID": user, - "USER_ID": user, - } - ) - if group: - result.update( - { - "PGID": group, - "GID": group, - "GROUP_ID": group, - } - ) - return result - - -def get_nvidia_env(gpus: dict) -> dict: - reservations = get_nvidia_gpus_reservations(gpus) - if not reservations.get("device_ids"): - return { - "NVIDIA_VISIBLE_DEVICES": "void", - } - - return { - "NVIDIA_VISIBLE_DEVICES": ( - ",".join(reservations["device_ids"]) - if reservations.get("device_ids") - else "void" - ), - "NVIDIA_DRIVER_CAPABILITIES": "all", - } diff --git a/ix-dev/stable/ix-app/templates/library/base_v1_1_7/healthchecks.py b/ix-dev/stable/ix-app/templates/library/base_v1_1_7/healthchecks.py deleted file mode 100644 index cc98270d1d..0000000000 --- a/ix-dev/stable/ix-app/templates/library/base_v1_1_7/healthchecks.py +++ /dev/null @@ -1,120 +0,0 @@ -from . import utils - - -def check_health(test, interval=10, timeout=5, retries=30, start_period=10): - if not test: - utils.throw_error("Expected [test] to be set") - - return { - "test": test, - "interval": f"{interval}s", - "timeout": f"{timeout}s", - "retries": retries, - "start_period": f"{start_period}s", - } - - -def mariadb_test(db, config=None): - config = config or {} - if not db: - utils.throw_error("MariaDB container: [db] must be set") - - host = config.get("host", "127.0.0.1") - port = config.get("port", 3306) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$$MARIADB_ROOT_PASSWORD ping" - - -def pg_test(user, db, config=None): - config = config or {} - if not user or not db: - utils.throw_error("Postgres container: [user] and [db] must be set") - - host = config.get("host", "127.0.0.1") - port = config.get("port", 5432) - - return f"pg_isready -h {host} -p {port} -d {db} -U {user}" - - -def redis_test(config=None): - config = config or {} - - host = config.get("host", "127.0.0.1") - port = config.get("port", 6379) - password = "$$REDIS_PASSWORD" - - return f"redis-cli -h {host} -p {port} -a {password} ping | grep -q PONG" - - -def curl_test(port, path, config=None): - config = config or {} - if not port or not path: - utils.throw_error("Expected [port] and [path] to be set") - - scheme = config.get("scheme", "http") - host = config.get("host", "127.0.0.1") - headers = config.get("headers", []) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - utils.throw_error("Expected [header] to be a list of two items") - opts.append(f'--header "{header[0]}: {header[1]}"') - - return f"curl --silent --output /dev/null --show-error --fail {' '.join(opts)} {scheme}://{host}:{port}{path}" - - -def wget_test(port, path, config=None): - config = config or {} - if not port or not path: - utils.throw_error("Expected [port] and [path] to be set") - - scheme = config.get("scheme", "http") - host = config.get("host", "127.0.0.1") - headers = config.get("headers", []) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - utils.throw_error("Expected [header] to be a list of two items") - opts.append(f'--header "{header[0]}: {header[1]}"') - - return f"wget --spider --quiet {' '.join(opts)} {scheme}://{host}:{port}{path}" - - -def http_test(port, path, config=None): - config = config or {} - if not port or not path: - utils.throw_error("Expected [port] and [path] to be set") - - host = config.get("host", "127.0.0.1") - - return f""" - /bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&$${{hc_fd}} && cat <&$${{hc_fd}} | grep "HTTP/1.1 200"' - """ # noqa - - -def netcat_test(port, config=None): - config = config or {} - if not port: - utils.throw_error("Expected [port] to be set") - - host = config.get("host", "127.0.0.1") - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(port, config=None): - config = config or {} - if not port: - utils.throw_error("Expected [port] to be set") - - host = config.get("host", "127.0.0.1") - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" diff --git a/ix-dev/stable/ix-app/templates/library/base_v1_1_7/mariadb.py b/ix-dev/stable/ix-app/templates/library/base_v1_1_7/mariadb.py deleted file mode 100644 index fc12daf81f..0000000000 --- a/ix-dev/stable/ix-app/templates/library/base_v1_1_7/mariadb.py +++ /dev/null @@ -1,72 +0,0 @@ -from . import utils -from .security import get_caps, get_sec_opts -from .network import dns_opts -from .healthchecks import mariadb_test, check_health -from .resources import resources - - -def mariadb_env(user, password, root_password, dbname): - if not user: - utils.throw_error("Expected [user] to be set for mariadb") - if not password: - utils.throw_error("Expected [password] to be set for mariadb") - if not root_password: - utils.throw_error("Expected [root_password] to be set for mariadb") - if not dbname: - utils.throw_error("Expected [dbname] to be set for mariadb") - return { - "MARIADB_USER": user, - "MARIADB_PASSWORD": utils.escape_dollar(password), - "MARIADB_ROOT_PASSWORD": utils.escape_dollar(root_password), - "MARIADB_DATABASE": dbname, - "MARIADB_AUTO_UPGRADE": "true", - } - - -def mariadb_container(data={}): - req_keys = [ - "db_user", - "db_password", - "db_root_password", - "db_name", - "volumes", - "resources", - ] - for key in req_keys: - if not data.get(key): - utils.throw_error(f"Expected [{key}] to be set for mariadb") - - db_user = data["db_user"] - db_password = data["db_password"] - db_root_password = data["db_root_password"] - db_name = data["db_name"] - db_port = data.get("port", 3306) - depends = data.get("depends_on", {}) - depends_on = {} - for key in depends: - depends_on[key] = { - "condition": depends[key].get("condition", "service_completed_successfully") - } - - return { - "image": f"{data.get('image', 'mariadb:10.6')}", - "user": f"{data.get('user', '999')}:{data.get('group', '999')}", - "restart": "unless-stopped", - "cap_drop": get_caps()["drop"], - "security_opt": get_sec_opts(), - **({"dns_opts": dns_opts(data["dns_opts"])} if data.get("dns_opts") else {}), - "healthcheck": check_health(mariadb_test(db=db_name, config={"port": db_port})), - "command": [ - "--port", - str(db_port), - ], - "environment": mariadb_env( - user=db_user, - password=db_password, - root_password=db_root_password, - dbname=db_name, - ), - "volumes": data["volumes"], - "depends_on": depends_on, - "deploy": {"resources": resources(data["resources"])}, - } diff --git a/ix-dev/stable/ix-app/templates/library/base_v1_1_7/metadata.py b/ix-dev/stable/ix-app/templates/library/base_v1_1_7/metadata.py deleted file mode 100644 index c0a59f8979..0000000000 --- a/ix-dev/stable/ix-app/templates/library/base_v1_1_7/metadata.py +++ /dev/null @@ -1,71 +0,0 @@ -from . import utils - - -def get_header(app_name: str): - return f"""# Welcome to TrueNAS SCALE - -Thank you for installing {app_name}! -""" - - -def get_footer(app_name: str): - return f"""## Documentation - -Documentation for {app_name} can be found at https://www.truenas.com/docs. - -## Bug reports - -If you find a bug in this app, please file an issue at -https://ixsystems.atlassian.net or https://github.com/truenas/apps - -## Feature requests or improvements - -If you find a feature request for this app, please file an issue at -https://ixsystems.atlassian.net or https://github.com/truenas/apps -""" - - -def get_notes(app_name: str, body: str = ""): - if not app_name: - utils.throw_error("Expected [app_name] to be set") - - return f"{get_header(app_name)}\n\n{body}\n\n{get_footer(app_name)}" - - -def get_portals(portals: list): - valid_schemes = ["http", "https"] - result = [] - for portal in portals: - # Most apps have a single portal, lets default to a standard name - name = portal.get("name", "Web UI") - scheme = portal.get("scheme", "http") - path = portal.get("path", "/") - - if not name: - utils.throw_error("Expected [portal.name] to be set") - if name in [p["name"] for p in result]: - utils.throw_error( - f"Expected [portal.name] to be unique, got [{', '.join([p['name'] for p in result]+[name])}]" - ) - if scheme not in valid_schemes: - utils.throw_error( - f"Expected [portal.scheme] to be one of [{', '.join(valid_schemes)}], got [{portal['scheme']}]" - ) - if not portal.get("port"): - utils.throw_error("Expected [portal.port] to be set") - if not path.startswith("/"): - utils.throw_error( - f"Expected [portal.path] to start with /, got [{portal['path']}]" - ) - - result.append( - { - "name": name, - "scheme": scheme, - "host": portal.get("host", "0.0.0.0"), - "port": portal["port"], - "path": path, - } - ) - - return result diff --git a/ix-dev/stable/ix-app/templates/library/base_v1_1_7/network.py b/ix-dev/stable/ix-app/templates/library/base_v1_1_7/network.py deleted file mode 100644 index e4761fd295..0000000000 --- a/ix-dev/stable/ix-app/templates/library/base_v1_1_7/network.py +++ /dev/null @@ -1,21 +0,0 @@ -from . import utils - - -def dns_opts(dns_options=None): - dns_options = dns_options or [] - if not dns_options: - return [] - - tracked = {} - disallowed_opts = [] - for opt in dns_options: - key = opt.split(":")[0] - if key in tracked: - utils.throw_error( - f"Expected [dns_opts] to be unique, got [{', '.join([d.split(':')[0] for d in tracked])}]" - ) - if key in disallowed_opts: - utils.throw_error(f"Expected [dns_opts] to not contain [{key}] key.") - tracked[key] = opt - - return dns_options diff --git a/ix-dev/stable/ix-app/templates/library/base_v1_1_7/permissions.py b/ix-dev/stable/ix-app/templates/library/base_v1_1_7/permissions.py deleted file mode 100644 index 1ce6e60ca1..0000000000 --- a/ix-dev/stable/ix-app/templates/library/base_v1_1_7/permissions.py +++ /dev/null @@ -1,139 +0,0 @@ -import jsonschema - -from . import utils - -ITEM_SCHEMA = { - "type": "object", - "properties": { - "dir": {"type": "string"}, - "mode": {"type": "string", "enum": ["always", "check"]}, - "uid": {"type": "integer"}, - "gid": {"type": "integer"}, - "chmod": {"type": "string"}, - "is_temporary": {"type": "boolean"}, - }, - "required": ["dir", "mode", "uid", "gid", "chmod", "is_temporary"], -} - - -def perms_container(items=[], volumes=[]): - if not items: - raise ValueError("Expected [items] to be set for perms_container") - if not volumes: - raise ValueError("Expected [volumes] to be set for perms_container") - - command = [process_dir_shell_func()] - for item in items: - try: - jsonschema.validate(item, ITEM_SCHEMA) - except jsonschema.ValidationError as e: - utils.throw_error(f"Item [{item}] is not valid: {e}") - cmd = [ - "process_dir", - item["dir"], - item["mode"], - str(item["uid"]), - str(item["gid"]), - item["chmod"], - str(item["is_temporary"]).lower(), - ] - command.append(" ".join(cmd)) - - return { - "image": "bash", - "user": "root", - "deploy": { - "resources": { - "limits": {"cpus": "1.0", "memory": "512m"}, - } - }, - "entrypoint": ["bash", "-c"], - "command": ["\n".join(command)], - "volumes": volumes, - } - - -# Don't forget to use double $ for shell variables, -# otherwise docker-compose will try to expand them -def process_dir_shell_func(): - return """ -function process_dir() { - local dir=$$1 - local mode=$$2 - local uid=$$3 - local gid=$$4 - local chmod=$$5 - local is_temporary=$$6 - - local fix_owner="false" - local fix_perms="false" - - if [ -z "$$dir" ]; then - echo "Path is empty, skipping..." - return 0 - fi - - if [ ! -d "$$dir" ]; then - echo "Path [$$dir] does is not a directory, skipping..." - return 0 - fi - - if [ "$$is_temporary" = "true" ]; then - echo "Path [$$dir] is a temporary directory, ensuring it is empty..." - # Exclude the safe directory, where we can use to mount files temporarily - find "$$dir" -mindepth 1 -maxdepth 1 ! -name "ix-safe" -exec rm -rf {} + - fi - - if [ "$$is_temporary" = "false" ] && [ -n "$$(ls -A $$dir)" ]; then - echo "Path [$$dir] is not empty, skipping..." - return 0 - fi - - echo "Current Ownership and Permissions on [$$dir]:" - echo "chown: $$(stat -c "%u %g" "$$dir")" - echo "chmod: $$(stat -c "%a" "$$dir")" - - if [ "$$mode" = "always" ]; then - fix_owner="true" - fix_perms="true" - fi - - if [ "$$mode" = "check" ]; then - if [ $$(stat -c %u "$$dir") -eq $$uid ] && [ $$(stat -c %g "$$dir") -eq $$gid ]; then - echo "Ownership is correct. Skipping..." - fix_owner="false" - else - echo "Ownership is incorrect. Fixing..." - fix_owner="true" - fi - - if [ "$$chmod" = "false" ]; then - echo "Skipping permissions check, chmod is false" - elif [ -n "$$chmod" ]; then - if [ $$(stat -c %a "$$dir") -eq $$chmod ]; then - echo "Permissions are correct. Skipping..." - fix_perms="false" - else - echo "Permissions are incorrect. Fixing..." - fix_perms="true" - fi - fi - fi - - if [ "$$fix_owner" = "true" ]; then - echo "Changing ownership to $$uid:$$gid on: [$$dir]" - chown -R "$$uid:$$gid" "$$dir" - echo "Finished changing ownership" - echo "Ownership after changes:" - stat -c "%u %g" "$$dir" - fi - - if [ -n "$$chmod" ] && [ "$$fix_perms" = "true" ]; then - echo "Changing permissions to $$chmod on: [$$dir]" - chmod -R "$$chmod" "$$dir" - echo "Finished changing permissions" - echo "Permissions after changes:" - stat -c "%a" "$$dir" - fi -} -""" diff --git a/ix-dev/stable/ix-app/templates/library/base_v1_1_7/ports.py b/ix-dev/stable/ix-app/templates/library/base_v1_1_7/ports.py deleted file mode 100644 index c895b47a44..0000000000 --- a/ix-dev/stable/ix-app/templates/library/base_v1_1_7/ports.py +++ /dev/null @@ -1,42 +0,0 @@ -import ipaddress - -from . import utils - - -def must_valid_port(num: int): - if num < 1 or num > 65535: - utils.throw_error(f"Expected a valid port number, got [{num}]") - - -def must_valid_ip(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - utils.throw_error(f"Expected a valid IP address, got [{ip}]") - - -def must_valid_protocol(protocol: str): - if protocol not in ["tcp", "udp"]: - utils.throw_error(f"Expected a valid protocol, got [{protocol}]") - - -def must_valid_mode(mode: str): - if mode not in ["ingress", "host"]: - utils.throw_error(f"Expected a valid mode, got [{mode}]") - - -def get_port(port=None): - port = port or {} - must_valid_port(port["published"]) - must_valid_port(port["target"]) - must_valid_ip(port.get("host_ip", "0.0.0.0")) - must_valid_protocol(port.get("protocol", "tcp")) - must_valid_mode(port.get("mode", "ingress")) - - return { - "target": port["target"], - "published": port["published"], - "protocol": port.get("protocol", "tcp"), - "mode": port.get("mode", "ingress"), - "host_ip": port.get("host_ip", "0.0.0.0"), - } diff --git a/ix-dev/stable/ix-app/templates/library/base_v1_1_7/postgres.py b/ix-dev/stable/ix-app/templates/library/base_v1_1_7/postgres.py deleted file mode 100644 index c5f8275454..0000000000 --- a/ix-dev/stable/ix-app/templates/library/base_v1_1_7/postgres.py +++ /dev/null @@ -1,77 +0,0 @@ -from . import utils -from .security import get_caps, get_sec_opts -from .network import dns_opts -from .healthchecks import pg_test, check_health -from .resources import resources - - -def pg_url(variant, host, user, password, dbname, port=5432): - if not host: - utils.throw_error("Expected [host] to be set") - if not user: - utils.throw_error("Expected [user] to be set") - if not password: - utils.throw_error("Expected [password] to be set") - if not dbname: - utils.throw_error("Expected [dbname] to be set") - - if variant == "postgresql": - return f"postgresql://{user}:{password}@{host}:{port}/{dbname}?sslmode=disable" - elif variant == "postgres": - return f"postgres://{user}:{password}@{host}:{port}/{dbname}?sslmode=disable" - else: - utils.throw_error( - f"Expected [variant] to be one of [postgresql, postgres], got [{variant}]" - ) - - -def pg_env(user, password, dbname, port=5432): - if not user: - utils.throw_error("Expected [user] to be set for postgres") - if not password: - utils.throw_error("Expected [password] to be set for postgres") - if not dbname: - utils.throw_error("Expected [dbname] to be set for postgres") - return { - "POSTGRES_USER": user, - "POSTGRES_PASSWORD": utils.escape_dollar(password), - "POSTGRES_DB": dbname, - "POSTGRES_PORT": port, - } - - -def pg_container(data={}): - req_keys = ["db_user", "db_password", "db_name", "volumes", "resources"] - for key in req_keys: - if not data.get(key): - utils.throw_error(f"Expected [{key}] to be set for postgres") - - pg_user = data["db_user"] - pg_password = data["db_password"] - pg_dbname = data["db_name"] - pg_port = data.get("port", 5432) - depends = data.get("depends_on", {}) - depends_on = {} - for key in depends: - depends_on[key] = { - "condition": depends[key].get("condition", "service_completed_successfully") - } - - return { - "image": f"{data.get('image', 'postgres:15')}", - "user": f"{data.get('user', '999')}:{data.get('group', '999')}", - "restart": "unless-stopped", - "cap_drop": get_caps()["drop"], - "security_opt": get_sec_opts(), - **({"dns_opts": dns_opts(data["dns_opts"])} if data.get("dns_opts") else {}), - "healthcheck": check_health(pg_test(user=pg_user, db=pg_dbname)), - "environment": pg_env( - user=pg_user, - password=pg_password, - dbname=pg_dbname, - port=pg_port, - ), - "volumes": data["volumes"], - "depends_on": depends_on, - "deploy": {"resources": resources(data["resources"])}, - } diff --git a/ix-dev/stable/ix-app/templates/library/base_v1_1_7/redis.py b/ix-dev/stable/ix-app/templates/library/base_v1_1_7/redis.py deleted file mode 100644 index 2356d0e06c..0000000000 --- a/ix-dev/stable/ix-app/templates/library/base_v1_1_7/redis.py +++ /dev/null @@ -1,49 +0,0 @@ -from . import utils -from .security import get_caps, get_sec_opts -from .network import dns_opts -from .healthchecks import redis_test, check_health -from .resources import resources - - -def redis_container(data={}): - req_keys = ["password", "volumes", "resources"] - for key in req_keys: - if not data.get(key): - utils.throw_error(f"Expected [{key}] to be set for postgres") - - redis_password = data["password"] - redis_port = data.get("port", 6379) - depends = data.get("depends_on", {}) - depends_on = {} - for key in depends: - depends_on[key] = { - "condition": depends[key].get("condition", "service_completed_successfully") - } - - return { - "image": f"{data.get('image', 'bitnami/redis:7.0.11')}", - "user": f"{data.get('user', '1001')}:{data.get('group', '0')}", - "restart": "unless-stopped", - "cap_drop": get_caps()["drop"], - "security_opt": get_sec_opts(), - **({"dns_opts": dns_opts(data["dns_opts"])} if data.get("dns_opts") else {}), - "healthcheck": check_health(redis_test(config={"port": redis_port})), - "environment": redis_env( - password=redis_password, - port=redis_port, - ), - "volumes": data["volumes"], - "depends_on": depends_on, - "deploy": {"resources": resources(data["resources"])}, - } - - -def redis_env(password, port=6379): - if not password: - utils.throw_error("Expected [password] to be set for redis") - - return { - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": utils.escape_dollar(password), - "REDIS_PORT_NUMBER": port, - } diff --git a/ix-dev/stable/ix-app/templates/library/base_v1_1_7/resources.py b/ix-dev/stable/ix-app/templates/library/base_v1_1_7/resources.py deleted file mode 100644 index d3235fd565..0000000000 --- a/ix-dev/stable/ix-app/templates/library/base_v1_1_7/resources.py +++ /dev/null @@ -1,101 +0,0 @@ -import re - -from . import utils - - -def resources(resources, disable_resource_limits=False): - gpus = resources.get("gpus", {}) - cpus = str(resources.get("limits", {}).get("cpus", 2.0)) - memory = str(resources.get("limits", {}).get("memory", 4096)) - if not re.match(r"^[1-9][0-9]*(\.[0-9]+)?$", cpus): - utils.throw_error(f"Expected cpus to be a number or a float, got [{cpus}]") - if not re.match(r"^[1-9][0-9]*$", memory): - raise ValueError(f"Expected memory to be a number, got [{memory}]") - - result = { - "limits": {"cpus": cpus, "memory": f"{memory}M"}, - "reservations": {"devices": []}, - } - - if gpus: - gpu_result = get_nvidia_gpus_reservations(gpus) - if gpu_result: - # Appending to devices, as we can later extend this to support other types of devices. Eg. TPUs. - result["reservations"]["devices"].append(get_nvidia_gpus_reservations(gpus)) - - # Docker does not like empty "things" all around. - if not result["reservations"]["devices"]: - del result["reservations"] - - if disable_resource_limits: - del result["limits"] - - return result - - -def get_nvidia_gpus_reservations(gpus: dict) -> dict: - """ - Input: - { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - """ - if not gpus: - return {} - - device_ids = [] - for pci, gpu in gpus.get("nvidia_gpu_selection", {}).items(): - if gpu["use_gpu"]: - if not gpu.get("uuid"): - utils.throw_error( - "Expected [uuid] to be set for GPU in" - f"slot [{pci}] in [nvidia_gpu_selection]" - ) - device_ids.append(gpu["uuid"]) - - if not device_ids: - return {} - - return { - "capabilities": ["gpu"], - "driver": "nvidia", - "device_ids": device_ids, - } - - -disallowed_devices = ["/dev/dri"] - - -# Returns the top level devices list -# Accepting other_devices to allow manually adding devices -# directly to the list. (Eg sound devices) -def get_devices(resources: dict, other_devices: list = []) -> list: - devices = [] - if resources.get("gpus", {}).get("use_all_gpus", False): - devices.append("/dev/dri:/dev/dri") - - added_host_devices: list = [] - for device in other_devices: - host_device = device.get("host_device", "").rstrip("/") - container_device = device.get("container_device", "") or host_device - if not host_device: - utils.throw_error(f"Expected [host_device] to be set for device [{device}]") - if not utils.valid_path(host_device): - utils.throw_error( - f"Expected [host_device] to be a valid path for device [{device}]" - ) - if host_device in disallowed_devices: - utils.throw_error( - f"Device [{host_device}] is not allowed to be manually added." - ) - if host_device in added_host_devices: - utils.throw_error( - f"Expected devices to be unique, but [{host_device}] was already added." - ) - devices.append(f"{host_device}:{container_device}") - added_host_devices.append(host_device) - - return devices diff --git a/ix-dev/stable/ix-app/templates/library/base_v1_1_7/security.py b/ix-dev/stable/ix-app/templates/library/base_v1_1_7/security.py deleted file mode 100644 index b67668b51b..0000000000 --- a/ix-dev/stable/ix-app/templates/library/base_v1_1_7/security.py +++ /dev/null @@ -1,34 +0,0 @@ -from base64 import b64encode - -from . import utils - - -def get_caps(add=None, drop=None): - add = add or [] - drop = drop or ["ALL"] - result = {"drop": drop} - if add: - result["add"] = add - return result - - -def get_sec_opts(add=None, remove=None): - add = add or [] - remove = remove or [] - result = ["no-new-privileges"] - for opt in add: - if opt not in result: - result.append(opt) - for opt in remove: - if opt in result: - result.remove(opt) - return result - - -def htpasswd(username, password): - hashed = utils.bcrypt_hash(password) - return username + ":" + hashed - - -def basic_auth(username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") diff --git a/ix-dev/stable/ix-app/templates/library/base_v1_1_7/storage.py b/ix-dev/stable/ix-app/templates/library/base_v1_1_7/storage.py deleted file mode 100644 index de09ba0c3e..0000000000 --- a/ix-dev/stable/ix-app/templates/library/base_v1_1_7/storage.py +++ /dev/null @@ -1,370 +0,0 @@ -import re -import json -import hashlib - -from . import utils - - -BIND_TYPES = ["host_path", "ix_volume"] -VOL_TYPES = ["volume", "nfs", "cifs", "temporary"] -ALL_TYPES = BIND_TYPES + VOL_TYPES + ["tmpfs", "anonymous"] -PROPAGATION_TYPES = ["shared", "slave", "private", "rshared", "rslave", "rprivate"] - - -def _get_name_for_temporary(data): - if not data.get("mount_path"): - utils.throw_error("Expected [mount_path] to be set for temporary volume") - - return data["mount_path"].lstrip("/").lower().replace("/", "_").replace(".", "_").replace(" ", "_") - - -# Returns a volume mount object (Used in container's "volumes" level) -def vol_mount(data, values=None): - values = values or {} - ix_volumes = values.get("ix_volumes") or [] - vol_type = _get_docker_vol_type(data) - - volume = { - "type": vol_type, - "target": utils.valid_path(data.get("mount_path", "")), - "read_only": data.get("read_only", False), - } - if vol_type == "bind": # Default create_host_path is true in short-syntax - volume.update(_get_bind_vol_config(data, values, ix_volumes)) - elif vol_type == "volume": - volume.update(_get_volume_vol_config(data)) - elif vol_type == "tmpfs": - volume.update(_get_tmpfs_vol_config(data)) - elif vol_type == "temporary": - volume["type"] = "volume" - volume.update(_get_volume_vol_config(data)) - elif vol_type == "anonymous": - volume["type"] = "volume" - volume.update(_get_anonymous_vol_config(data)) - - return volume - - -def storage_item(data, values=None, perm_opts=None): - values = values or {} - perm_opts = perm_opts or {} - if data.get("type") == "temporary": - data.update({"volume_name": _get_name_for_temporary(data)}) - return { - "vol_mount": vol_mount(data, values), - "vol": vol(data), - "perms_item": perms_item(data, values, perm_opts) if perm_opts else {}, - } - - -def perms_item(data, values=None, opts=None): - opts = opts or {} - values = values or {} - vol_type = data.get("type", "") - - # Temp volumes are always auto permissions - if vol_type == "temporary": - data.update({"auto_permissions": True}) - - # If its ix_volume, we need to set auto permissions - if vol_type == "ix_volume": - data.update({"auto_permissions": True}) - - if not data.get("auto_permissions"): - return {} - - if vol_type == "host_path": - if data.get("host_path_config", {}).get("acl_enable", False): - return {} - if vol_type == "ix_volume": - if data.get("ix_volume_config", {}).get("acl_enable", False): - return {} - - req_keys = ["mount_path", "mode", "uid", "gid"] - for key in req_keys: - if opts.get(key, None) is None: - utils.throw_error(f"Expected opts passed to [perms_item] to have [{key}] key") - - data.update({"mount_path": opts["mount_path"]}) - volume_mount = vol_mount(data, values) - # For perms volume mount, always set read_only to false - volume_mount.update({"read_only": False}) - - return { - "vol_mount": volume_mount, - "perm_dir": { - "dir": volume_mount["target"], - "mode": opts["mode"], - "uid": opts["uid"], - "gid": opts["gid"], - "chmod": opts.get("chmod", "false"), - "is_temporary": data["type"] == "temporary", - }, - } - - -def create_host_path_default(values): - """ - By default, do not create host path for bind mounts if it does not exist. - If the ix_context is missing, we are either in local dev or CI. - We should create the host path by default there to ease development. - The _magic_ "dev_mode" flag is added so we can also toggle this behavior - in CI, while we are also using ix_context for other tests. - """ - ix_ctx = values.get("ix_context", {}) - if not ix_ctx: - return True - if "dev_mode" in ix_ctx: - return ix_ctx["dev_mode"] - return False - - -def _get_bind_vol_config(data, values, ix_volumes=None): - ix_volumes = ix_volumes or [] - path = host_path(data, ix_volumes) - if data.get("propagation", "rprivate") not in PROPAGATION_TYPES: - utils.throw_error( - f"Expected [propagation] to be one of [{', '.join(PROPAGATION_TYPES)}], got [{data['propagation']}]" - ) - - # https://docs.docker.com/storage/bind-mounts/#configure-bind-propagation - return { - "source": path, - "bind": { - "create_host_path": data.get("host_path_config", {}).get( - "create_host_path", create_host_path_default(values) - ), - "propagation": _get_valid_propagation(data), - }, - } - - -def _get_volume_vol_config(data): - if data.get("type") in ["nfs", "cifs"]: - if data.get("volume_name"): - utils.throw_error("Expected [volume_name] to be empty for [nfs, cifs] type") - data.update({"volume_name": _get_name_for_external_volume(data)}) - if not data.get("volume_name"): - utils.throw_error("Expected [volume_name] to be set for [volume] type") - - return {"source": data["volume_name"], "volume": _process_volume_config(data)} - - -def _get_anonymous_vol_config(data): - return {"volume": _process_volume_config(data)} - - -mode_regex = re.compile(r"^0[0-7]{3}$") - - -def _get_tmpfs_vol_config(data): - tmpfs = {} - config = data.get("tmpfs_config", {}) - - if config.get("size"): - if not isinstance(config["size"], int): - utils.throw_error("Expected [size] to be an integer for [tmpfs] type") - if not config["size"] > 0: - utils.throw_error("Expected [size] to be greater than 0 for [tmpfs] type") - # Convert Mebibytes to Bytes - tmpfs.update({"size": config["size"] * 1024 * 1024}) - - if config.get("mode"): - if not mode_regex.match(str(config["mode"])): - utils.throw_error(f"Expected [mode] to be a octal string for [tmpfs] type, got [{config['mode']}]") - tmpfs.update({"mode": int(config["mode"], 8)}) - - return {"tmpfs": tmpfs} - - -# We generate a unique name for the volume based on the config -# Docker will not update any volume after creation. This is to ensure -# that changing any value (eg server address) in the config will result in a new volume -def _get_name_for_external_volume(data): - config_hash = hashlib.sha256(json.dumps(data).encode("utf-8")).hexdigest() - return f"{data['type']}_{config_hash}" - - -# Returns a volume object (Used in top "volumes" level) -def vol(data): - if not data or _get_docker_vol_type(data) != "volume": - return {} - - if not data.get("volume_name"): - utils.throw_error("Expected [volume_name] to be set for [volume] type") - - if data["type"] == "nfs": - return {data["volume_name"]: _process_nfs(data)} - elif data["type"] == "cifs": - return {data["volume_name"]: _process_cifs(data)} - else: - return {data["volume_name"]: {}} - - -def _is_host_path(data): - return data.get("type") == "host_path" - - -def _get_valid_propagation(data): - if not data.get("propagation"): - return "rprivate" - if not data["propagation"] in PROPAGATION_TYPES: - utils.throw_error( - f"Expected [propagation] to be one of [{', '.join(PROPAGATION_TYPES)}], got [{data['propagation']}]" - ) - return data["propagation"] - - -def _is_ix_volume(data): - return data.get("type") == "ix_volume" - - -# Returns the host path for a for either a host_path or ix_volume -def host_path(data, ix_volumes=None): - ix_volumes = ix_volumes or [] - path = "" - if _is_host_path(data): - path = _process_host_path_config(data) - elif _is_ix_volume(data): - path = _process_ix_volume_config(data, ix_volumes) - else: - utils.throw_error( - f"Expected [host_path()] to be called only for types [host_path, ix_volume], got [{data['type']}]" - ) - - return utils.valid_path(path) - - -# Returns the type of storage as used in docker-compose -def _get_docker_vol_type(data): - if not data.get("type"): - utils.throw_error("Expected [type] to be set for storage") - - if data["type"] not in ALL_TYPES: - utils.throw_error(f"Expected storage [type] to be one of {ALL_TYPES}, got [{data['type']}]") - - if data["type"] in BIND_TYPES: - return "bind" - elif data["type"] in VOL_TYPES: - return "volume" - else: - return data["type"] - - -def _process_host_path_config(data): - if data.get("host_path_config", {}).get("acl_enable", False): - if not data["host_path_config"].get("acl", {}).get("path"): - utils.throw_error("Expected [host_path_config.acl.path] to be set for [host_path] type with ACL enabled") - return data["host_path_config"]["acl"]["path"] - - if not data.get("host_path_config", {}).get("path"): - utils.throw_error("Expected [host_path_config.path] to be set for [host_path] type") - - return data["host_path_config"]["path"] - - -def _process_volume_config(data): - return {"nocopy": data.get("volume_config", {}).get("nocopy", False)} - - -def _process_ix_volume_config(data, ix_volumes): - path = "" - if not data.get("ix_volume_config", {}).get("dataset_name"): - utils.throw_error("Expected [ix_volume_config.dataset_name] to be set for [ix_volume] type") - - if not ix_volumes: - utils.throw_error("Expected [ix_volumes] to be set for [ix_volume] type") - - ds = data["ix_volume_config"]["dataset_name"] - path = ix_volumes.get(ds, None) - if not path: - utils.throw_error(f"Expected the key [{ds}] to be set in [ix_volumes]") - - return path - - -# Constructs a volume object for a cifs type -def _process_cifs(data): - if not data.get("cifs_config"): - utils.throw_error("Expected [cifs_config] to be set for [cifs] type") - - required_keys = ["server", "path", "username", "password"] - for key in required_keys: - if not data["cifs_config"].get(key): - utils.throw_error(f"Expected [{key}] to be set for [cifs] type") - - opts = [ - "noperm", - f"user={data['cifs_config']['username']}", - f"password={data['cifs_config']['password']}", - ] - if data["cifs_config"].get("domain"): - opts.append(f'domain={data["cifs_config"]["domain"]}') - - if data["cifs_config"].get("options"): - if not isinstance(data["cifs_config"]["options"], list): - utils.throw_error("Expected [cifs_config.options] to be a list for [cifs] type") - - disallowed_opts = ["user", "password", "domain", "noperm"] - for opt in data["cifs_config"]["options"]: - if not isinstance(opt, str): - utils.throw_error("Expected [cifs_config.options] to be a list of strings for [cifs] type") - - key = opt.split("=")[0] - for disallowed in disallowed_opts: - if key == disallowed: - utils.throw_error( - f"Expected [cifs_config.options] to not start with [{disallowed}] for [cifs] type" - ) - - opts.append(opt) - - server = data["cifs_config"]["server"].lstrip("/") - path = data["cifs_config"]["path"].strip("/") - volume = { - "driver_opts": { - "type": "cifs", - "device": f"//{server}/{path}", - "o": f"{','.join([utils.escape_dollar(opt) for opt in opts])}", - }, - } - - return volume - - -# Constructs a volume object for a nfs type -def _process_nfs(data): - if not data.get("nfs_config"): - utils.throw_error("Expected [nfs_config] to be set for [nfs] type") - - required_keys = ["server", "path"] - for key in required_keys: - if not data["nfs_config"].get(key): - utils.throw_error(f"Expected [{key}] to be set for [nfs] type") - - opts = [f"addr={data['nfs_config']['server']}"] - if data["nfs_config"].get("options"): - if not isinstance(data["nfs_config"]["options"], list): - utils.throw_error("Expected [nfs_config.options] to be a list for [nfs] type") - - disallowed_opts = ["addr"] - for opt in data["nfs_config"]["options"]: - if not isinstance(opt, str): - utils.throw_error("Expected [nfs_config.options] to be a list of strings for [nfs] type") - - key = opt.split("=")[0] - for disallowed in disallowed_opts: - if key == disallowed: - utils.throw_error(f"Expected [nfs_config.options] to not start with [{disallowed}] for [nfs] type") - - opts.append(opt) - - volume = { - "driver_opts": { - "type": "nfs", - "device": f":{data['nfs_config']['path']}", - "o": f"{','.join([utils.escape_dollar(opt) for opt in opts])}", - }, - } - - return volume diff --git a/ix-dev/stable/ix-app/templates/library/base_v1_1_7/utils.py b/ix-dev/stable/ix-app/templates/library/base_v1_1_7/utils.py deleted file mode 100644 index 8a7c0815c6..0000000000 --- a/ix-dev/stable/ix-app/templates/library/base_v1_1_7/utils.py +++ /dev/null @@ -1,124 +0,0 @@ -import hashlib -import secrets -import bcrypt -import sys -import re - -from . import security - - -class TemplateException(Exception): - pass - - -def throw_error(message): - # When throwing a known error, hide the traceback - # This is because the error is also shown in the UI - # and having a traceback makes it hard for user to read - sys.tracebacklimit = 0 - raise TemplateException(message) - - -def secure_string(length): - return secrets.token_urlsafe(length) - - -def basic_auth_header(username, password): - return f"Basic {security.basic_auth(username, password)}" - - -def bcrypt_hash(password): - return bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - - -def match_regex(value, regex): - if not re.match(regex, value): - return False - return True - - -def must_match_regex(value, regex): - if not match_regex(value, regex): - throw_error(f"Expected [{value}] to match [{regex}]") - return value - - -def merge_dicts(*dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - -# Basic validation for a path (Expand later) -def valid_path(path=""): - if not path.startswith("/"): - throw_error(f"Expected path [{path}] to start with /") - - # There is no reason to allow / as a path, either on host or in a container - if path == "/": - throw_error(f"Expected path [{path}] to not be /") - - return path - - -def camel_case(string): - return string.title() - - -def is_boolean(string): - return string.lower() in ["true", "false"] - - -def is_number(string): - try: - float(string) - return True - except ValueError: - return False - - -def get_image(images={}, name=""): - if not images: - throw_error("Expected [images] to be set") - if name not in images: - throw_error(f"Expected [images.{name}] to be set") - if not images[name].get("repository") or not images[name].get("tag"): - throw_error(f"Expected [images.{name}.repository] and [images.{name}.tag] to be set") - - return f"{images[name]['repository']}:{images[name]['tag']}" - - -def hash_data(data=""): - if not data: - throw_error("Expected [data] to be set") - return hashlib.sha256(data.encode("utf-8")).hexdigest() - - -def get_image_with_hashed_data(images={}, name="", data=""): - return f"ix-{get_image(images, name)}-{hash_data(data)}" - - -def copy_dict(dict): - return dict.copy() - - -def escape_dollar(text: str) -> str: - return text.replace("$", "$$") - - -def auto_cast(value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value diff --git a/ix-dev/stable/ix-app/templates/library/base_v1_1_7/__init__.py b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/__init__.py similarity index 100% rename from ix-dev/stable/ix-app/templates/library/base_v1_1_7/__init__.py rename to ix-dev/stable/ix-app/templates/library/base_v2_0_34/__init__.py diff --git a/library/2.0.33/configs.py b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/configs.py similarity index 100% rename from library/2.0.33/configs.py rename to ix-dev/stable/ix-app/templates/library/base_v2_0_34/configs.py diff --git a/library/2.0.33/container.py b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/container.py similarity index 93% rename from library/2.0.33/container.py rename to ix-dev/stable/ix-app/templates/library/base_v2_0_34/container.py index a95e76734c..701f64bfeb 100644 --- a/library/2.0.33/container.py +++ b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/container.py @@ -17,7 +17,7 @@ from .labels import Labels from .ports import Ports from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise from .storage import Storage from .sysctls import Sysctls except ImportError: @@ -33,7 +33,7 @@ from labels import Labels from ports import Ports from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise from storage import Storage from sysctls import Sysctls @@ -45,6 +45,7 @@ def __init__(self, render_instance: "Render", name: str, image: str): self._name: str = name self._image: str = self._resolve_image(image) self._build_image: str = "" + self._pull_policy: str = "" self._user: str = "" self._tty: bool = False self._stdin_open: bool = False @@ -54,6 +55,7 @@ def __init__(self, render_instance: "Render", name: str, image: str): self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly self._cap_add: set[str] = set() self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False self._group_add: set[int | str] = set() self._network_mode: str = "" self._entrypoint: list[str] = [] @@ -130,6 +132,9 @@ def build_image(self, content: list[str | None]): self._build_image = dockerfile self._image = get_image_with_hashed_data(self._image, dockerfile) + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + def set_user(self, user: int, group: int): for i in (user, group): if not isinstance(i, int) or i < 0: @@ -169,6 +174,13 @@ def set_grace_period(self, grace_period: int): raise RenderError(f"Grace period [{grace_period}] cannot be negative") self._grace_period = grace_period + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + def add_caps(self, caps: list[str]): for c in caps: if c in self._cap_add: @@ -230,10 +242,14 @@ def render(self) -> dict[str, Any]: "tty": self._tty, "stdin_open": self._stdin_open, "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), } + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + if self._hostname: result["hostname"] = self._hostname @@ -265,6 +281,12 @@ def render(self) -> dict[str, Any]: if self._shm_size is not None: result["shm_size"] = f"{self._shm_size}M" + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + if self._cap_add: result["cap_add"] = sorted(self._cap_add) diff --git a/library/2.0.33/depends.py b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/depends.py similarity index 100% rename from library/2.0.33/depends.py rename to ix-dev/stable/ix-app/templates/library/base_v2_0_34/depends.py diff --git a/library/2.0.33/deploy.py b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/deploy.py similarity index 100% rename from library/2.0.33/deploy.py rename to ix-dev/stable/ix-app/templates/library/base_v2_0_34/deploy.py diff --git a/library/2.0.33/deps.py b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/deps.py similarity index 100% rename from library/2.0.33/deps.py rename to ix-dev/stable/ix-app/templates/library/base_v2_0_34/deps.py diff --git a/library/2.0.33/device.py b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/device.py similarity index 100% rename from library/2.0.33/device.py rename to ix-dev/stable/ix-app/templates/library/base_v2_0_34/device.py diff --git a/library/2.0.33/devices.py b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/devices.py similarity index 100% rename from library/2.0.33/devices.py rename to ix-dev/stable/ix-app/templates/library/base_v2_0_34/devices.py diff --git a/library/2.0.33/dns.py b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/dns.py similarity index 100% rename from library/2.0.33/dns.py rename to ix-dev/stable/ix-app/templates/library/base_v2_0_34/dns.py diff --git a/library/2.0.33/environment.py b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/environment.py similarity index 95% rename from library/2.0.33/environment.py rename to ix-dev/stable/ix-app/templates/library/base_v2_0_34/environment.py index 850a3afd8e..056763ea80 100644 --- a/library/2.0.33/environment.py +++ b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/environment.py @@ -23,10 +23,13 @@ def __init__(self, render_instance: "Render", resources: Resources): # Stores variables that are added by the application developer self._app_dev_variables: dict[str, Any] = {} + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + self._auto_add_variables_from_values() def _auto_add_variables_from_values(self): - self._add_generic_variables() + if not self._skip_generic_variables: + self._add_generic_variables() self._add_nvidia_variables() def _add_generic_variables(self): diff --git a/library/2.0.33/error.py b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/error.py similarity index 100% rename from library/2.0.33/error.py rename to ix-dev/stable/ix-app/templates/library/base_v2_0_34/error.py diff --git a/library/2.0.33/formatter.py b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/formatter.py similarity index 100% rename from library/2.0.33/formatter.py rename to ix-dev/stable/ix-app/templates/library/base_v2_0_34/formatter.py diff --git a/library/2.0.33/functions.py b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/functions.py similarity index 100% rename from library/2.0.33/functions.py rename to ix-dev/stable/ix-app/templates/library/base_v2_0_34/functions.py diff --git a/library/2.0.33/healthcheck.py b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/healthcheck.py similarity index 95% rename from library/2.0.33/healthcheck.py rename to ix-dev/stable/ix-app/templates/library/base_v2_0_34/healthcheck.py index 36ae5d90aa..0805329284 100644 --- a/library/2.0.33/healthcheck.py +++ b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/healthcheck.py @@ -22,6 +22,7 @@ def __init__(self, render_instance: "Render"): self._retries: int = 30 self._start_period_sec: int = 10 self._disabled: bool = False + self._use_built_in: bool = False def _get_test(self): if isinstance(self._test, str): @@ -32,6 +33,9 @@ def _get_test(self): def disable(self): self._disabled = True + def use_built_in(self): + self._use_built_in = True + def set_custom_test(self, test: str | list[str]): if self._disabled: raise RenderError("Cannot set custom test when healthcheck is disabled") @@ -53,7 +57,13 @@ def set_retries(self, retries: int): def set_start_period(self, start_period: int): self._start_period_sec = start_period + def has_healthcheck(self): + return not self._use_built_in + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + if self._disabled: return {"disable": True} diff --git a/library/2.0.33/labels.py b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/labels.py similarity index 100% rename from library/2.0.33/labels.py rename to ix-dev/stable/ix-app/templates/library/base_v2_0_34/labels.py diff --git a/library/2.0.33/notes.py b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/notes.py similarity index 100% rename from library/2.0.33/notes.py rename to ix-dev/stable/ix-app/templates/library/base_v2_0_34/notes.py diff --git a/library/2.0.33/portal.py b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/portal.py similarity index 100% rename from library/2.0.33/portal.py rename to ix-dev/stable/ix-app/templates/library/base_v2_0_34/portal.py diff --git a/library/2.0.33/portals.py b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/portals.py similarity index 100% rename from library/2.0.33/portals.py rename to ix-dev/stable/ix-app/templates/library/base_v2_0_34/portals.py diff --git a/library/2.0.33/ports.py b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/ports.py similarity index 100% rename from library/2.0.33/ports.py rename to ix-dev/stable/ix-app/templates/library/base_v2_0_34/ports.py diff --git a/library/2.0.33/render.py b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/render.py similarity index 100% rename from library/2.0.33/render.py rename to ix-dev/stable/ix-app/templates/library/base_v2_0_34/render.py diff --git a/library/2.0.33/resources.py b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/resources.py similarity index 100% rename from library/2.0.33/resources.py rename to ix-dev/stable/ix-app/templates/library/base_v2_0_34/resources.py diff --git a/library/2.0.33/restart.py b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/restart.py similarity index 100% rename from library/2.0.33/restart.py rename to ix-dev/stable/ix-app/templates/library/base_v2_0_34/restart.py diff --git a/library/2.0.33/storage.py b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/storage.py similarity index 100% rename from library/2.0.33/storage.py rename to ix-dev/stable/ix-app/templates/library/base_v2_0_34/storage.py diff --git a/library/2.0.33/sysctls.py b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/sysctls.py similarity index 100% rename from library/2.0.33/sysctls.py rename to ix-dev/stable/ix-app/templates/library/base_v2_0_34/sysctls.py diff --git a/library/2.0.33/__init__.py b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/tests/__init__.py similarity index 100% rename from library/2.0.33/__init__.py rename to ix-dev/stable/ix-app/templates/library/base_v2_0_34/tests/__init__.py diff --git a/library/2.0.33/tests/test_build_image.py b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/tests/test_build_image.py similarity index 100% rename from library/2.0.33/tests/test_build_image.py rename to ix-dev/stable/ix-app/templates/library/base_v2_0_34/tests/test_build_image.py diff --git a/library/2.0.33/tests/test_configs.py b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/tests/test_configs.py similarity index 100% rename from library/2.0.33/tests/test_configs.py rename to ix-dev/stable/ix-app/templates/library/base_v2_0_34/tests/test_configs.py diff --git a/library/2.0.33/tests/test_container.py b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/tests/test_container.py similarity index 89% rename from library/2.0.33/tests/test_container.py rename to ix-dev/stable/ix-app/templates/library/base_v2_0_34/tests/test_container.py index 61a22a5df2..747ad39357 100644 --- a/library/2.0.33/tests/test_container.py +++ b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/tests/test_container.py @@ -50,6 +50,42 @@ def test_non_existing_image(mock_values): render.add_container("test_container", "non_existing_image") +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + def test_tty(mock_values): render = Render(mock_values) c1 = render.add_container("test_container", "test_image") diff --git a/library/2.0.33/tests/test_depends.py b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/tests/test_depends.py similarity index 100% rename from library/2.0.33/tests/test_depends.py rename to ix-dev/stable/ix-app/templates/library/base_v2_0_34/tests/test_depends.py diff --git a/library/2.0.33/tests/test_deps.py b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/tests/test_deps.py similarity index 100% rename from library/2.0.33/tests/test_deps.py rename to ix-dev/stable/ix-app/templates/library/base_v2_0_34/tests/test_deps.py diff --git a/library/2.0.33/tests/test_device.py b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/tests/test_device.py similarity index 100% rename from library/2.0.33/tests/test_device.py rename to ix-dev/stable/ix-app/templates/library/base_v2_0_34/tests/test_device.py diff --git a/library/2.0.33/tests/test_dns.py b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/tests/test_dns.py similarity index 100% rename from library/2.0.33/tests/test_dns.py rename to ix-dev/stable/ix-app/templates/library/base_v2_0_34/tests/test_dns.py diff --git a/library/2.0.33/tests/test_environment.py b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/tests/test_environment.py similarity index 93% rename from library/2.0.33/tests/test_environment.py rename to ix-dev/stable/ix-app/templates/library/base_v2_0_34/tests/test_environment.py index 209f67551b..d657646582 100644 --- a/library/2.0.33/tests/test_environment.py +++ b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/tests/test_environment.py @@ -47,6 +47,18 @@ def test_auto_add_vars(mock_values): assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + def test_add_from_all_sources(mock_values): mock_values["TZ"] = "Etc/UTC" render = Render(mock_values) diff --git a/library/2.0.33/tests/test_formatter.py b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/tests/test_formatter.py similarity index 100% rename from library/2.0.33/tests/test_formatter.py rename to ix-dev/stable/ix-app/templates/library/base_v2_0_34/tests/test_formatter.py diff --git a/library/2.0.33/tests/test_functions.py b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/tests/test_functions.py similarity index 100% rename from library/2.0.33/tests/test_functions.py rename to ix-dev/stable/ix-app/templates/library/base_v2_0_34/tests/test_functions.py diff --git a/library/2.0.33/tests/test_healthcheck.py b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/tests/test_healthcheck.py similarity index 95% rename from library/2.0.33/tests/test_healthcheck.py rename to ix-dev/stable/ix-app/templates/library/base_v2_0_34/tests/test_healthcheck.py index fbd488ece4..8fa044290f 100644 --- a/library/2.0.33/tests/test_healthcheck.py +++ b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/tests/test_healthcheck.py @@ -23,6 +23,14 @@ def test_disable_healthcheck(mock_values): assert output["services"]["test_container"]["healthcheck"] == {"disable": True} +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + def test_set_custom_test(mock_values): render = Render(mock_values) c1 = render.add_container("test_container", "test_image") diff --git a/library/2.0.33/tests/test_labels.py b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/tests/test_labels.py similarity index 100% rename from library/2.0.33/tests/test_labels.py rename to ix-dev/stable/ix-app/templates/library/base_v2_0_34/tests/test_labels.py diff --git a/library/2.0.33/tests/test_notes.py b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/tests/test_notes.py similarity index 100% rename from library/2.0.33/tests/test_notes.py rename to ix-dev/stable/ix-app/templates/library/base_v2_0_34/tests/test_notes.py diff --git a/library/2.0.33/tests/test_portal.py b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/tests/test_portal.py similarity index 100% rename from library/2.0.33/tests/test_portal.py rename to ix-dev/stable/ix-app/templates/library/base_v2_0_34/tests/test_portal.py diff --git a/library/2.0.33/tests/test_ports.py b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/tests/test_ports.py similarity index 100% rename from library/2.0.33/tests/test_ports.py rename to ix-dev/stable/ix-app/templates/library/base_v2_0_34/tests/test_ports.py diff --git a/library/2.0.33/tests/test_render.py b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/tests/test_render.py similarity index 100% rename from library/2.0.33/tests/test_render.py rename to ix-dev/stable/ix-app/templates/library/base_v2_0_34/tests/test_render.py diff --git a/library/2.0.33/tests/test_resources.py b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/tests/test_resources.py similarity index 100% rename from library/2.0.33/tests/test_resources.py rename to ix-dev/stable/ix-app/templates/library/base_v2_0_34/tests/test_resources.py diff --git a/library/2.0.33/tests/test_restart.py b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/tests/test_restart.py similarity index 100% rename from library/2.0.33/tests/test_restart.py rename to ix-dev/stable/ix-app/templates/library/base_v2_0_34/tests/test_restart.py diff --git a/library/2.0.33/tests/test_sysctls.py b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/tests/test_sysctls.py similarity index 100% rename from library/2.0.33/tests/test_sysctls.py rename to ix-dev/stable/ix-app/templates/library/base_v2_0_34/tests/test_sysctls.py diff --git a/library/2.0.33/tests/test_volumes.py b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/tests/test_volumes.py similarity index 100% rename from library/2.0.33/tests/test_volumes.py rename to ix-dev/stable/ix-app/templates/library/base_v2_0_34/tests/test_volumes.py diff --git a/library/2.0.33/validations.py b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/validations.py similarity index 95% rename from library/2.0.33/validations.py rename to ix-dev/stable/ix-app/templates/library/base_v2_0_34/validations.py index b0dea174b7..43dd96b475 100644 --- a/library/2.0.33/validations.py +++ b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/validations.py @@ -9,6 +9,13 @@ OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + def valid_sysctl_or_raise(sysctl: str, host_network: bool): if not sysctl: raise RenderError("Sysctl cannot be empty") diff --git a/library/2.0.33/volume_mount.py b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/volume_mount.py similarity index 100% rename from library/2.0.33/volume_mount.py rename to ix-dev/stable/ix-app/templates/library/base_v2_0_34/volume_mount.py diff --git a/library/2.0.33/volume_mount_types.py b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/volume_mount_types.py similarity index 100% rename from library/2.0.33/volume_mount_types.py rename to ix-dev/stable/ix-app/templates/library/base_v2_0_34/volume_mount_types.py diff --git a/library/2.0.33/volume_sources.py b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/volume_sources.py similarity index 100% rename from library/2.0.33/volume_sources.py rename to ix-dev/stable/ix-app/templates/library/base_v2_0_34/volume_sources.py diff --git a/library/2.0.33/volume_types.py b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/volume_types.py similarity index 100% rename from library/2.0.33/volume_types.py rename to ix-dev/stable/ix-app/templates/library/base_v2_0_34/volume_types.py diff --git a/library/2.0.33/volumes.py b/ix-dev/stable/ix-app/templates/library/base_v2_0_34/volumes.py similarity index 100% rename from library/2.0.33/volumes.py rename to ix-dev/stable/ix-app/templates/library/base_v2_0_34/volumes.py diff --git a/ix-dev/stable/ix-app/templates/test_values/basic-values.yaml b/ix-dev/stable/ix-app/templates/test_values/basic-values.yaml index b104cf748a..840ae46633 100644 --- a/ix-dev/stable/ix-app/templates/test_values/basic-values.yaml +++ b/ix-dev/stable/ix-app/templates/test_values/basic-values.yaml @@ -8,13 +8,18 @@ image: pull_policy: missing # Container Configuration -restart_policy: "no" +restart_policy: "on-failure" +max_retry_count: 0 entrypoint: [] command: [] TZ: "Europe/Athens" envs: - name: test value: 123 + - name: UMASK + value: "002" + - name: PGID + value: "568" disable_builtin_healthcheck: false tty: false stdin: false @@ -36,6 +41,11 @@ dns_config: searches: [] options: [] +# General Configuration +labels: + - key: com.truenas.ix.app.labels.test + value: 123 + # Portal Configuration portals: - name: Web UI @@ -43,21 +53,33 @@ portals: use_node_ip: true host: "" port: 80 - path: "/" + path: / - name: Admin UI protocol: http use_node_ip: false - host: "192.168.1.100" + host: 192.168.1.100 port: 80 - path: "/" + path: / + +ix_volumes: + some_volume: /opt/tests/mnt/test_volume # Storage Configuration storage: + - type: ix_volume + mount_path: /opt/tests/mnt/test_volume + ix_volume_config: + dataset_name: some_volume + create_host_path: true + - type: tmpfs + mount_path: /opt/tests/mnt/test_tmpfs + tmpfs_config: + size: 500 - type: host_path - mount_path: /tmp + mount_path: /opt/tests/mnt/test host_path_config: - path: /tmp - read_only: false + path: /test + create_host_path: true # Resources Configuration resources: diff --git a/library/2.0.33/tests/__init__.py b/library/2.0.34/__init__.py similarity index 100% rename from library/2.0.33/tests/__init__.py rename to library/2.0.34/__init__.py diff --git a/library/2.0.34/configs.py b/library/2.0.34/configs.py new file mode 100644 index 0000000000..b76f4b169c --- /dev/null +++ b/library/2.0.34/configs.py @@ -0,0 +1,86 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class Configs: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._configs: dict[str, dict] = {} + + def add(self, name: str, data: str): + if not isinstance(data, str): + raise RenderError(f"Expected [data] to be a string, got [{type(data)}]") + + if name not in self._configs: + self._configs[name] = {"name": name, "data": data} + return + + if data == self._configs[name]["data"]: + return + + raise RenderError(f"Config [{name}] already added with different data") + + def has_configs(self): + return bool(self._configs) + + def render(self): + return { + c["name"]: {"content": escape_dollar(c["data"])} + for c in sorted(self._configs.values(), key=lambda c: c["name"]) + } + + +class ContainerConfigs: + def __init__(self, render_instance: "Render", configs: Configs): + self._render_instance = render_instance + self.top_level_configs: Configs = configs + self.container_configs: set[ContainerConfig] = set() + + def add(self, name: str, data: str, target: str, mode: str = ""): + self.top_level_configs.add(name, data) + + if target == "": + raise RenderError(f"Expected [target] to be set for config [{name}]") + if mode != "": + mode = valid_octal_mode_or_raise(mode) + + if target in [c.target for c in self.container_configs]: + raise RenderError(f"Target [{target}] already used for another config") + target = valid_fs_path_or_raise(target) + self.container_configs.add(ContainerConfig(self._render_instance, name, target, mode)) + + def has_configs(self): + return bool(self.container_configs) + + def render(self): + return [c.render() for c in sorted(self.container_configs, key=lambda c: c.source)] + + +class ContainerConfig: + def __init__(self, render_instance: "Render", source: str, target: str, mode: str): + self._render_instance = render_instance + self.source = source + self.target = target + self.mode = mode + + def render(self): + result: dict[str, str | int] = { + "source": self.source, + "target": self.target, + } + + if self.mode: + result["mode"] = int(self.mode, 8) + + return result diff --git a/library/2.0.34/container.py b/library/2.0.34/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/library/2.0.34/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/library/2.0.34/depends.py b/library/2.0.34/depends.py new file mode 100644 index 0000000000..4e057cf085 --- /dev/null +++ b/library/2.0.34/depends.py @@ -0,0 +1,34 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_depend_condition_or_raise +except ImportError: + from error import RenderError + from validations import valid_depend_condition_or_raise + + +class Depends: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._dependencies: dict[str, str] = {} + + def add_dependency(self, name: str, condition: str): + condition = valid_depend_condition_or_raise(condition) + if name in self._dependencies.keys(): + raise RenderError(f"Dependency [{name}] already added") + if name not in self._render_instance.container_names(): + raise RenderError( + f"Dependency [{name}] not found in defined containers. " + f"Available containers: [{', '.join(self._render_instance.container_names())}]" + ) + self._dependencies[name] = condition + + def has_dependencies(self): + return len(self._dependencies) > 0 + + def render(self): + return {d: {"condition": c} for d, c in self._dependencies.items()} diff --git a/library/2.0.34/deploy.py b/library/2.0.34/deploy.py new file mode 100644 index 0000000000..894dbc643b --- /dev/null +++ b/library/2.0.34/deploy.py @@ -0,0 +1,24 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .resources import Resources +except ImportError: + from resources import Resources + + +class Deploy: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self.resources: Resources = Resources(self._render_instance) + + def has_deploy(self): + return self.resources.has_resources() + + def render(self): + if self.resources.has_resources(): + return {"resources": self.resources.render()} + + return {} diff --git a/library/2.0.34/deps.py b/library/2.0.34/deps.py new file mode 100644 index 0000000000..b3607fa6ab --- /dev/null +++ b/library/2.0.34/deps.py @@ -0,0 +1,454 @@ +import os +import json +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import ( + valid_port_or_raise, + valid_fs_path_or_raise, + valid_octal_mode_or_raise, + valid_redis_password_or_raise, + ) +except ImportError: + from error import RenderError + from validations import ( + valid_port_or_raise, + valid_fs_path_or_raise, + valid_octal_mode_or_raise, + valid_redis_password_or_raise, + ) + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = os.path.join("/mnt/permission", identifier) + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + + c.add_storage("/var/lib/postgresql/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("POSTGRES_USER", config["user"]) + c.environment.add_env("POSTGRES_PASSWORD", config["password"]) + c.environment.add_env("POSTGRES_DB", config["database"]) + c.environment.add_env("POSTGRES_PORT", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def _get_port(self): + return self._config.get("port") or 5432 + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/library/2.0.34/device.py b/library/2.0.34/device.py new file mode 100644 index 0000000000..bfe97097cb --- /dev/null +++ b/library/2.0.34/device.py @@ -0,0 +1,31 @@ +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise, allowed_device_or_raise, valid_cgroup_perm_or_raise +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise, allowed_device_or_raise, valid_cgroup_perm_or_raise + + +class Device: + def __init__(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + hd = valid_fs_path_or_raise(host_device.rstrip("/")) + cd = valid_fs_path_or_raise(container_device.rstrip("/")) + if not hd or not cd: + raise RenderError( + "Expected [host_device] and [container_device] to be set. " + f"Got host_device [{host_device}] and container_device [{container_device}]" + ) + + cgroup_perm = valid_cgroup_perm_or_raise(cgroup_perm) + if not allow_disallowed: + hd = allowed_device_or_raise(hd) + + self.cgroup_perm: str = cgroup_perm + self.host_device: str = hd + self.container_device: str = cd + + def render(self): + result = f"{self.host_device}:{self.container_device}" + if self.cgroup_perm: + result += f":{self.cgroup_perm}" + return result diff --git a/library/2.0.34/devices.py b/library/2.0.34/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/library/2.0.34/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/library/2.0.34/dns.py b/library/2.0.34/dns.py new file mode 100644 index 0000000000..d3ae7b19fa --- /dev/null +++ b/library/2.0.34/dns.py @@ -0,0 +1,79 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import allowed_dns_opt_or_raise +except ImportError: + from error import RenderError + from validations import allowed_dns_opt_or_raise + + +class Dns: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._dns_options: set[str] = set() + self._dns_searches: set[str] = set() + self._dns_nameservers: set[str] = set() + + self._auto_add_dns_opts_from_values() + self._auto_add_dns_searches_from_values() + self._auto_add_dns_nameservers_from_values() + + def _get_dns_opt_keys(self): + return [self._get_key_from_opt(opt) for opt in self._dns_options] + + def _get_key_from_opt(self, opt): + return opt.split(":")[0] + + def _auto_add_dns_opts_from_values(self): + values = self._render_instance.values + for dns_opt in values.get("network", {}).get("dns_opts", []): + self.add_dns_opt(dns_opt) + + def _auto_add_dns_searches_from_values(self): + values = self._render_instance.values + for dns_search in values.get("network", {}).get("dns_searches", []): + self.add_dns_search(dns_search) + + def _auto_add_dns_nameservers_from_values(self): + values = self._render_instance.values + for dns_nameserver in values.get("network", {}).get("dns_nameservers", []): + self.add_dns_nameserver(dns_nameserver) + + def add_dns_search(self, dns_search): + if dns_search in self._dns_searches: + raise RenderError(f"DNS Search [{dns_search}] already added") + self._dns_searches.add(dns_search) + + def add_dns_nameserver(self, dns_nameserver): + if dns_nameserver in self._dns_nameservers: + raise RenderError(f"DNS Nameserver [{dns_nameserver}] already added") + self._dns_nameservers.add(dns_nameserver) + + def add_dns_opt(self, dns_opt): + # eg attempts:3 + key = allowed_dns_opt_or_raise(self._get_key_from_opt(dns_opt)) + if key in self._get_dns_opt_keys(): + raise RenderError(f"DNS Option [{key}] already added") + self._dns_options.add(dns_opt) + + def has_dns_opts(self): + return len(self._dns_options) > 0 + + def has_dns_searches(self): + return len(self._dns_searches) > 0 + + def has_dns_nameservers(self): + return len(self._dns_nameservers) > 0 + + def render_dns_searches(self): + return sorted(self._dns_searches) + + def render_dns_opts(self): + return sorted(self._dns_options) + + def render_dns_nameservers(self): + return sorted(self._dns_nameservers) diff --git a/library/2.0.34/environment.py b/library/2.0.34/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/library/2.0.34/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/library/2.0.34/error.py b/library/2.0.34/error.py new file mode 100644 index 0000000000..aef48d3b02 --- /dev/null +++ b/library/2.0.34/error.py @@ -0,0 +1,4 @@ +class RenderError(Exception): + """Base class for exceptions in this module.""" + + pass diff --git a/library/2.0.34/formatter.py b/library/2.0.34/formatter.py new file mode 100644 index 0000000000..24e882f47a --- /dev/null +++ b/library/2.0.34/formatter.py @@ -0,0 +1,26 @@ +import json +import hashlib + + +def escape_dollar(text: str) -> str: + return text.replace("$", "$$") + + +def get_hashed_name_for_volume(prefix: str, config: dict): + config_hash = hashlib.sha256(json.dumps(config).encode("utf-8")).hexdigest() + return f"{prefix}_{config_hash}" + + +def get_hash_with_prefix(prefix: str, data: str): + return f"{prefix}_{hashlib.sha256(data.encode('utf-8')).hexdigest()}" + + +def merge_dicts_no_overwrite(dict1, dict2): + overlapping_keys = dict1.keys() & dict2.keys() + if overlapping_keys: + raise ValueError(f"Merging of dicts failed. Overlapping keys: {overlapping_keys}") + return {**dict1, **dict2} + + +def get_image_with_hashed_data(image: str, data: str): + return get_hash_with_prefix(f"ix-{image}", data) diff --git a/library/2.0.34/functions.py b/library/2.0.34/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/library/2.0.34/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/library/2.0.34/healthcheck.py b/library/2.0.34/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/library/2.0.34/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/library/2.0.34/labels.py b/library/2.0.34/labels.py new file mode 100644 index 0000000000..f1e667ba00 --- /dev/null +++ b/library/2.0.34/labels.py @@ -0,0 +1,37 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar +except ImportError: + from error import RenderError + from formatter import escape_dollar + + +class Labels: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._labels: dict[str, str] = {} + + def add_label(self, key: str, value: str): + if not key: + raise RenderError("Labels must have a key") + + if key.startswith("com.docker.compose"): + raise RenderError(f"Label [{key}] cannot start with [com.docker.compose] as it is reserved") + + if key in self._labels.keys(): + raise RenderError(f"Label [{key}] already added") + + self._labels[key] = escape_dollar(str(value)) + + def has_labels(self) -> bool: + return bool(self._labels) + + def render(self) -> dict[str, str]: + if not self.has_labels(): + return {} + return {label: value for label, value in sorted(self._labels.items())} diff --git a/library/2.0.34/notes.py b/library/2.0.34/notes.py new file mode 100644 index 0000000000..4adc50c3d8 --- /dev/null +++ b/library/2.0.34/notes.py @@ -0,0 +1,70 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + + +class Notes: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._app_name: str = "" + self._warnings: list[str] = [] + self._deprecations: list[str] = [] + self._header: str = "" + self._body: str = "" + self._footer: str = "" + + self._auto_set_app_name() + self._auto_set_header() + self._auto_set_footer() + + def _auto_set_app_name(self): + app_name = self._render_instance.values.get("ix_context", {}).get("app_metadata", {}).get("name", "") + self._app_name = app_name or "" + + def _auto_set_header(self): + head = "# Welcome to TrueNAS SCALE\n\n" + head += f"Thank you for installing {self._app_name}!\n\n" + self._header = head + + def _auto_set_footer(self): + footer = "## Documentation\n\n" + footer += f"Documentation for {self._app_name} can be found at https://www.truenas.com/docs.\n\n" + footer += "## Bug reports\n\n" + footer += "If you find a bug in this app, please file an issue at\n" + footer += "https://ixsystems.atlassian.net or https://github.com/truenas/apps\n\n" + footer += "## Feature requests or improvements\n\n" + footer += "If you find a feature request for this app, please file an issue at\n" + footer += "https://ixsystems.atlassian.net or https://github.com/truenas/apps\n" + self._footer = footer + + def add_warning(self, warning: str): + self._warnings.append(warning) + + def add_deprecation(self, deprecation: str): + self._deprecations.append(deprecation) + + def set_body(self, body: str): + self._body = body + + def render(self): + result = self._header + + if self._warnings: + result += "## Warnings\n\n" + for warning in self._warnings: + result += f"- {warning}\n" + result += "\n" + + if self._deprecations: + result += "## Deprecations\n\n" + for deprecation in self._deprecations: + result += f"- {deprecation}\n" + result += "\n" + + if self._body: + result += self._body.strip() + "\n\n" + + result += self._footer + + return result diff --git a/library/2.0.34/portal.py b/library/2.0.34/portal.py new file mode 100644 index 0000000000..cf47163439 --- /dev/null +++ b/library/2.0.34/portal.py @@ -0,0 +1,22 @@ +try: + from .validations import valid_portal_scheme_or_raise, valid_http_path_or_raise, valid_port_or_raise +except ImportError: + from validations import valid_portal_scheme_or_raise, valid_http_path_or_raise, valid_port_or_raise + + +class Portal: + def __init__(self, name: str, config: dict): + self._name = name + self._scheme = valid_portal_scheme_or_raise(config.get("scheme", "http")) + self._host = config.get("host", "0.0.0.0") or "0.0.0.0" + self._port = valid_port_or_raise(config.get("port", 0)) + self._path = valid_http_path_or_raise(config.get("path", "/")) + + def render(self): + return { + "name": self._name, + "scheme": self._scheme, + "host": self._host, + "port": self._port, + "path": self._path, + } diff --git a/library/2.0.34/portals.py b/library/2.0.34/portals.py new file mode 100644 index 0000000000..e106d231e6 --- /dev/null +++ b/library/2.0.34/portals.py @@ -0,0 +1,28 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .portal import Portal +except ImportError: + from error import RenderError + from portal import Portal + + +class Portals: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._portals: set[Portal] = set() + + def add_portal(self, config: dict): + name = config.get("name", "Web UI") + + if name in [p._name for p in self._portals]: + raise RenderError(f"Portal [{name}] already added") + + self._portals.add(Portal(name, config)) + + def render(self): + return [p.render() for _, p in sorted([(p._name, p) for p in self._portals])] diff --git a/library/2.0.34/ports.py b/library/2.0.34/ports.py new file mode 100644 index 0000000000..f11e1481b4 --- /dev/null +++ b/library/2.0.34/ports.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import ( + valid_port_or_raise, + valid_port_protocol_or_raise, + valid_port_mode_or_raise, + valid_ip_or_raise, + ) +except ImportError: + from error import RenderError + from validations import ( + valid_port_or_raise, + valid_port_protocol_or_raise, + valid_port_mode_or_raise, + valid_ip_or_raise, + ) + + +class Ports: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._ports: dict[str, dict] = {} + + def add_port(self, host_port: int, container_port: int, config: dict | None = None): + config = config or {} + host_port = valid_port_or_raise(host_port) + container_port = valid_port_or_raise(container_port) + proto = valid_port_protocol_or_raise(config.get("protocol", "tcp")) + mode = valid_port_mode_or_raise(config.get("mode", "ingress")) + host_ip = valid_ip_or_raise(config.get("host_ip", "0.0.0.0")) + + key = f"{host_port}_{host_ip}_{proto}" + if key in self._ports.keys(): + raise RenderError(f"Port [{host_port}/{proto}] already added for [{host_ip}]") + + if host_ip != "0.0.0.0": + # If the port we are adding is not going to use 0.0.0.0 + # Make sure that we don't have already added that port/proto to 0.0.0.0 + search_key = f"{host_port}_0.0.0.0_{proto}" + if search_key in self._ports.keys(): + raise RenderError(f"Cannot bind port [{host_port}/{proto}] to [{host_ip}], already bound to [0.0.0.0]") + elif host_ip == "0.0.0.0": + # If the port we are adding is going to use 0.0.0.0 + # Make sure that we don't have already added that port/proto to a specific ip + for p in self._ports.values(): + if p["published"] == host_port and p["protocol"] == proto: + raise RenderError( + f"Cannot bind port [{host_port}/{proto}] to [{host_ip}], already bound to [{p['host_ip']}]" + ) + + self._ports[key] = { + "published": host_port, + "target": container_port, + "protocol": proto, + "mode": mode, + "host_ip": host_ip, + } + + def has_ports(self): + return len(self._ports) > 0 + + def render(self): + return [config for _, config in sorted(self._ports.items())] diff --git a/library/2.0.34/render.py b/library/2.0.34/render.py new file mode 100644 index 0000000000..9d8fcc28d5 --- /dev/null +++ b/library/2.0.34/render.py @@ -0,0 +1,89 @@ +import copy + +try: + from .configs import Configs + from .container import Container + from .deps import Deps + from .error import RenderError + from .functions import Functions + from .notes import Notes + from .portals import Portals + from .volumes import Volumes +except ImportError: + from configs import Configs + from container import Container + from deps import Deps + from error import RenderError + from functions import Functions + from notes import Notes + from portals import Portals + from volumes import Volumes + + +class Render(object): + def __init__(self, values): + self._containers: dict[str, Container] = {} + self.values = values + self._add_images_internal_use() + # Make a copy after we inject the images + self._original_values: dict = copy.deepcopy(self.values) + + self.deps: Deps = Deps(self) + + self.configs = Configs(render_instance=self) + self.funcs = Functions(render_instance=self).func_map() + self.portals: Portals = Portals(render_instance=self) + self.notes: Notes = Notes(render_instance=self) + self.volumes = Volumes(render_instance=self) + + def _add_images_internal_use(self): + if not self.values.get("images"): + self.values["images"] = {} + + if "python_permissions_image" not in self.values["images"]: + self.values["images"]["python_permissions_image"] = {"repository": "python", "tag": "3.13.0-slim-bookworm"} + + def container_names(self): + return list(self._containers.keys()) + + def add_container(self, name: str, image: str): + name = name.strip() + if not name: + raise RenderError("Container name cannot be empty") + container = Container(self, name, image) + if name in self._containers: + raise RenderError(f"Container {name} already exists.") + self._containers[name] = container + return container + + def render(self): + if self.values != self._original_values: + raise RenderError("Values have been modified since the renderer was created.") + + if not self._containers: + raise RenderError("No containers added.") + + result: dict = { + "x-notes": self.notes.render(), + "x-portals": self.portals.render(), + "services": {c._name: c.render() for c in self._containers.values()}, + } + + # Make sure that after services are rendered + # there are no labels that target a non-existent container + # This is to prevent typos + for label in self.values.get("labels", []): + for c in label.get("containers", []): + if c not in self.container_names(): + raise RenderError(f"Label [{label['key']}] references container [{c}] which does not exist") + + if self.volumes.has_volumes(): + result["volumes"] = self.volumes.render() + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + # if self.networks: + # result["networks"] = {...} + + return result diff --git a/library/2.0.34/resources.py b/library/2.0.34/resources.py new file mode 100644 index 0000000000..733f43bb6f --- /dev/null +++ b/library/2.0.34/resources.py @@ -0,0 +1,115 @@ +import re +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +DEFAULT_CPUS = 2.0 +DEFAULT_MEMORY = 4096 + + +class Resources: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._limits: dict = {} + self._reservations: dict = {} + self._nvidia_ids: set[str] = set() + self._auto_add_cpu_from_values() + self._auto_add_memory_from_values() + self._auto_add_gpus_from_values() + + def _set_cpu(self, cpus: Any): + c = str(cpus) + if not re.match(r"^[1-9][0-9]*(\.[0-9]+)?$", c): + raise RenderError(f"Expected cpus to be a number or a float (minimum 1.0), got [{cpus}]") + self._limits.update({"cpus": c}) + + def _set_memory(self, memory: Any): + m = str(memory) + if not re.match(r"^[1-9][0-9]*$", m): + raise RenderError(f"Expected memory to be a number, got [{memory}]") + self._limits.update({"memory": f"{m}M"}) + + def _auto_add_cpu_from_values(self): + resources = self._render_instance.values.get("resources", {}) + self._set_cpu(resources.get("limits", {}).get("cpus", DEFAULT_CPUS)) + + def _auto_add_memory_from_values(self): + resources = self._render_instance.values.get("resources", {}) + self._set_memory(resources.get("limits", {}).get("memory", DEFAULT_MEMORY)) + + def _auto_add_gpus_from_values(self): + resources = self._render_instance.values.get("resources", {}) + gpus = resources.get("gpus", {}).get("nvidia_gpu_selection", {}) + if not gpus: + return + + for pci, gpu in gpus.items(): + if gpu.get("use_gpu", False): + if not gpu.get("uuid"): + raise RenderError(f"Expected [uuid] to be set for GPU in slot [{pci}] in [nvidia_gpu_selection]") + self._nvidia_ids.add(gpu["uuid"]) + + if self._nvidia_ids: + if not self._reservations: + self._reservations["devices"] = [] + self._reservations["devices"].append( + { + "capabilities": ["gpu"], + "driver": "nvidia", + "device_ids": sorted(self._nvidia_ids), + } + ) + + # This is only used on ix-app that we allow + # disabling cpus and memory. GPUs are only added + # if the user has requested them. + def remove_cpus_and_memory(self): + self._limits.pop("cpus", None) + self._limits.pop("memory", None) + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._reservations.pop("devices", None) + + def set_profile(self, profile: str): + cpu, memory = profile_mapping(profile) + self._set_cpu(cpu) + self._set_memory(memory) + + def has_resources(self): + return len(self._limits) > 0 or len(self._reservations) > 0 + + def has_gpus(self): + gpu_devices = [d for d in self._reservations.get("devices", []) if "gpu" in d["capabilities"]] + return len(gpu_devices) > 0 + + def render(self): + result = {} + if self._limits: + result["limits"] = self._limits + if self._reservations: + result["reservations"] = self._reservations + + return result + + +def profile_mapping(profile: str): + profiles = { + "low": (1, 512), + "medium": (2, 1024), + } + + if profile not in profiles: + raise RenderError( + f"Resource profile [{profile}] is not valid. Valid options are: [{', '.join(profiles.keys())}]" + ) + + return profiles[profile] diff --git a/library/2.0.34/restart.py b/library/2.0.34/restart.py new file mode 100644 index 0000000000..2f6281af48 --- /dev/null +++ b/library/2.0.34/restart.py @@ -0,0 +1,25 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .validations import valid_restart_policy_or_raise +except ImportError: + from validations import valid_restart_policy_or_raise + + +class RestartPolicy: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._policy: str = "unless-stopped" + self._maximum_retry_count: int = 0 + + def set_policy(self, policy: str, maximum_retry_count: int = 0): + self._policy = valid_restart_policy_or_raise(policy, maximum_retry_count) + self._maximum_retry_count = maximum_retry_count + + def render(self): + if self._policy == "on-failure" and self._maximum_retry_count > 0: + return f"{self._policy}:{self._maximum_retry_count}" + return self._policy diff --git a/library/2.0.34/storage.py b/library/2.0.34/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/library/2.0.34/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/library/2.0.34/sysctls.py b/library/2.0.34/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/library/2.0.34/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/library/2.0.34/tests/__init__.py b/library/2.0.34/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/library/2.0.34/tests/test_build_image.py b/library/2.0.34/tests/test_build_image.py new file mode 100644 index 0000000000..f30c1210ed --- /dev/null +++ b/library/2.0.34/tests/test_build_image.py @@ -0,0 +1,49 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_build_image_with_from(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.build_image(["FROM test_image"]) + + +def test_build_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.build_image( + [ + "RUN echo hello", + None, + "", + "RUN echo world", + ] + ) + output = render.render() + assert ( + output["services"]["test_container"]["image"] + == "ix-nginx:latest_4a127145ea4c25511707e57005dd0ed457fe2f4932082c8f9faa339a450b6a99" + ) + assert output["services"]["test_container"]["build"] == { + "tags": ["ix-nginx:latest_4a127145ea4c25511707e57005dd0ed457fe2f4932082c8f9faa339a450b6a99"], + "dockerfile_inline": """FROM nginx:latest +RUN echo hello +RUN echo world +""", + } diff --git a/library/2.0.34/tests/test_configs.py b/library/2.0.34/tests/test_configs.py new file mode 100644 index 0000000000..9049e473ea --- /dev/null +++ b/library/2.0.34/tests/test_configs.py @@ -0,0 +1,63 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_duplicate_config_with_different_data(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.configs.add("test_config", "test_data", "/some/path") + with pytest.raises(Exception): + c1.configs.add("test_config", "test_data2", "/some/path") + + +def test_add_config_with_empty_target(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.configs.add("test_config", "test_data", "") + + +def test_add_duplicate_target(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.configs.add("test_config", "test_data", "/some/path") + with pytest.raises(Exception): + c1.configs.add("test_config2", "test_data2", "/some/path") + + +def test_add_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.configs.add("test_config", "$test_data", "/some/path") + output = render.render() + assert output["configs"]["test_config"]["content"] == "$$test_data" + assert output["services"]["test_container"]["configs"] == [{"source": "test_config", "target": "/some/path"}] + + +def test_add_config_with_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.configs.add("test_config", "test_data", "/some/path", "0777") + output = render.render() + assert output["configs"]["test_config"]["content"] == "test_data" + assert output["services"]["test_container"]["configs"] == [ + {"source": "test_config", "target": "/some/path", "mode": 511} + ] diff --git a/library/2.0.34/tests/test_container.py b/library/2.0.34/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/library/2.0.34/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/library/2.0.34/tests/test_depends.py b/library/2.0.34/tests/test_depends.py new file mode 100644 index 0000000000..a1d8373927 --- /dev/null +++ b/library/2.0.34/tests/test_depends.py @@ -0,0 +1,54 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_dependency(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c2 = render.add_container("test_container2", "test_image") + c1.healthcheck.disable() + c2.healthcheck.disable() + c1.depends.add_dependency("test_container2", "service_started") + output = render.render() + assert output["services"]["test_container"]["depends_on"]["test_container2"] == {"condition": "service_started"} + + +def test_add_dependency_invalid_condition(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + render.add_container("test_container2", "test_image") + with pytest.raises(Exception): + c1.depends.add_dependency("test_container2", "invalid_condition") + + +def test_add_dependency_missing_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.depends.add_dependency("test_container2", "service_started") + + +def test_add_dependency_duplicate(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + render.add_container("test_container2", "test_image") + c1.depends.add_dependency("test_container2", "service_started") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.depends.add_dependency("test_container2", "service_started") diff --git a/library/2.0.34/tests/test_deps.py b/library/2.0.34/tests/test_deps.py new file mode 100644 index 0000000000..f9562ba4f2 --- /dev/null +++ b/library/2.0.34/tests/test_deps.py @@ -0,0 +1,380 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:latest" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] diff --git a/library/2.0.34/tests/test_device.py b/library/2.0.34/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/library/2.0.34/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/library/2.0.34/tests/test_dns.py b/library/2.0.34/tests/test_dns.py new file mode 100644 index 0000000000..fe6b21e34f --- /dev/null +++ b/library/2.0.34/tests/test_dns.py @@ -0,0 +1,64 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_dns_opts(mock_values): + mock_values["network"] = {"dns_opts": ["attempts:3", "opt1", "opt2"]} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["dns_opt"] == ["attempts:3", "opt1", "opt2"] + + +def test_auto_add_dns_searches(mock_values): + mock_values["network"] = {"dns_searches": ["search1", "search2"]} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["dns_search"] == ["search1", "search2"] + + +def test_auto_add_dns_nameservers(mock_values): + mock_values["network"] = {"dns_nameservers": ["nameserver1", "nameserver2"]} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["dns"] == ["nameserver1", "nameserver2"] + + +def test_add_duplicate_dns_nameservers(mock_values): + mock_values["network"] = {"dns_nameservers": ["nameserver1", "nameserver1"]} + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_add_duplicate_dns_searches(mock_values): + mock_values["network"] = {"dns_searches": ["search1", "search1"]} + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_add_duplicate_dns_opts(mock_values): + mock_values["network"] = {"dns_opts": ["attempts:3", "attempts:5"]} + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") diff --git a/library/2.0.34/tests/test_environment.py b/library/2.0.34/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/library/2.0.34/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/library/2.0.34/tests/test_formatter.py b/library/2.0.34/tests/test_formatter.py new file mode 100644 index 0000000000..843cf65d2e --- /dev/null +++ b/library/2.0.34/tests/test_formatter.py @@ -0,0 +1,13 @@ +from formatter import escape_dollar + + +def test_escape_dollar(): + cases = [ + {"input": "test", "expected": "test"}, + {"input": "$test", "expected": "$$test"}, + {"input": "$$test", "expected": "$$$$test"}, + {"input": "$$$test", "expected": "$$$$$$test"}, + {"input": "$test$", "expected": "$$test$$"}, + ] + for case in cases: + assert escape_dollar(case["input"]) == case["expected"] diff --git a/library/2.0.34/tests/test_functions.py b/library/2.0.34/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/library/2.0.34/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/library/2.0.34/tests/test_healthcheck.py b/library/2.0.34/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/library/2.0.34/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/library/2.0.34/tests/test_labels.py b/library/2.0.34/tests/test_labels.py new file mode 100644 index 0000000000..ffa21eceac --- /dev/null +++ b/library/2.0.34/tests/test_labels.py @@ -0,0 +1,88 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_disallowed_label(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.labels.add_label("com.docker.compose.service", "test_service") + + +def test_add_duplicate_label(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.labels.add_label("my.custom.label", "test_value") + with pytest.raises(Exception): + c1.labels.add_label("my.custom.label", "test_value1") + + +def test_add_label_on_non_existing_container(mock_values): + mock_values["labels"] = [ + { + "key": "my.custom.label1", + "value": "test_value1", + "containers": ["test_container", "test_container2"], + }, + ] + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.render() + + +def test_add_label(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.labels.add_label("my.custom.label1", "test_value1") + c1.labels.add_label("my.custom.label2", "test_value2") + output = render.render() + assert output["services"]["test_container"]["labels"] == { + "my.custom.label1": "test_value1", + "my.custom.label2": "test_value2", + } + + +def test_auto_add_labels(mock_values): + mock_values["labels"] = [ + { + "key": "my.custom.label1", + "value": "test_value1", + "containers": ["test_container", "test_container2"], + }, + { + "key": "my.custom.label2", + "value": "test_value2", + "containers": ["test_container"], + }, + ] + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c2 = render.add_container("test_container2", "test_image") + c1.healthcheck.disable() + c2.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["labels"] == { + "my.custom.label1": "test_value1", + "my.custom.label2": "test_value2", + } + assert output["services"]["test_container2"]["labels"] == { + "my.custom.label1": "test_value1", + } diff --git a/library/2.0.34/tests/test_notes.py b/library/2.0.34/tests/test_notes.py new file mode 100644 index 0000000000..3613445385 --- /dev/null +++ b/library/2.0.34/tests/test_notes.py @@ -0,0 +1,213 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "ix_context": { + "app_metadata": { + "name": "test_app", + } + }, + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_notes(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert ( + output["x-notes"] + == """# Welcome to TrueNAS SCALE + +Thank you for installing test_app! + +## Documentation + +Documentation for test_app can be found at https://www.truenas.com/docs. + +## Bug reports + +If you find a bug in this app, please file an issue at +https://ixsystems.atlassian.net or https://github.com/truenas/apps + +## Feature requests or improvements + +If you find a feature request for this app, please file an issue at +https://ixsystems.atlassian.net or https://github.com/truenas/apps +""" + ) + + +def test_notes_with_warnings(mock_values): + render = Render(mock_values) + render.notes.add_warning("this is not properly configured. fix it now!") + render.notes.add_warning("that is not properly configured. fix it later!") + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert ( + output["x-notes"] + == """# Welcome to TrueNAS SCALE + +Thank you for installing test_app! + +## Warnings + +- this is not properly configured. fix it now! +- that is not properly configured. fix it later! + +## Documentation + +Documentation for test_app can be found at https://www.truenas.com/docs. + +## Bug reports + +If you find a bug in this app, please file an issue at +https://ixsystems.atlassian.net or https://github.com/truenas/apps + +## Feature requests or improvements + +If you find a feature request for this app, please file an issue at +https://ixsystems.atlassian.net or https://github.com/truenas/apps +""" + ) + + +def test_notes_with_deprecations(mock_values): + render = Render(mock_values) + render.notes.add_deprecation("this is will be removed later. fix it now!") + render.notes.add_deprecation("that is will be removed later. fix it later!") + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert ( + output["x-notes"] + == """# Welcome to TrueNAS SCALE + +Thank you for installing test_app! + +## Deprecations + +- this is will be removed later. fix it now! +- that is will be removed later. fix it later! + +## Documentation + +Documentation for test_app can be found at https://www.truenas.com/docs. + +## Bug reports + +If you find a bug in this app, please file an issue at +https://ixsystems.atlassian.net or https://github.com/truenas/apps + +## Feature requests or improvements + +If you find a feature request for this app, please file an issue at +https://ixsystems.atlassian.net or https://github.com/truenas/apps +""" + ) + + +def test_notes_with_body(mock_values): + render = Render(mock_values) + render.notes.set_body( + """## Additional info + +Some info +some other info. +""" + ) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert ( + output["x-notes"] + == """# Welcome to TrueNAS SCALE + +Thank you for installing test_app! + +## Additional info + +Some info +some other info. + +## Documentation + +Documentation for test_app can be found at https://www.truenas.com/docs. + +## Bug reports + +If you find a bug in this app, please file an issue at +https://ixsystems.atlassian.net or https://github.com/truenas/apps + +## Feature requests or improvements + +If you find a feature request for this app, please file an issue at +https://ixsystems.atlassian.net or https://github.com/truenas/apps +""" + ) + + +def test_notes_all(mock_values): + render = Render(mock_values) + render.notes.add_warning("this is not properly configured. fix it now!") + render.notes.add_warning("that is not properly configured. fix it later!") + render.notes.add_deprecation("this is will be removed later. fix it now!") + render.notes.add_deprecation("that is will be removed later. fix it later!") + render.notes.set_body( + """## Additional info + +Some info +some other info. +""" + ) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert ( + output["x-notes"] + == """# Welcome to TrueNAS SCALE + +Thank you for installing test_app! + +## Warnings + +- this is not properly configured. fix it now! +- that is not properly configured. fix it later! + +## Deprecations + +- this is will be removed later. fix it now! +- that is will be removed later. fix it later! + +## Additional info + +Some info +some other info. + +## Documentation + +Documentation for test_app can be found at https://www.truenas.com/docs. + +## Bug reports + +If you find a bug in this app, please file an issue at +https://ixsystems.atlassian.net or https://github.com/truenas/apps + +## Feature requests or improvements + +If you find a feature request for this app, please file an issue at +https://ixsystems.atlassian.net or https://github.com/truenas/apps +""" + ) diff --git a/library/2.0.34/tests/test_portal.py b/library/2.0.34/tests/test_portal.py new file mode 100644 index 0000000000..aebd9425c9 --- /dev/null +++ b/library/2.0.34/tests/test_portal.py @@ -0,0 +1,75 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_no_portals(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["x-portals"] == [] + + +def test_add_portal(mock_values): + render = Render(mock_values) + render.portals.add_portal({"scheme": "http", "path": "/", "port": 8080}) + render.portals.add_portal({"name": "Other Portal", "scheme": "https", "path": "/", "port": 8443}) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["x-portals"] == [ + {"name": "Other Portal", "scheme": "https", "host": "0.0.0.0", "port": 8443, "path": "/"}, + {"name": "Web UI", "scheme": "http", "host": "0.0.0.0", "port": 8080, "path": "/"}, + ] + + +def test_add_duplicate_portal(mock_values): + render = Render(mock_values) + render.portals.add_portal({"scheme": "http", "path": "/", "port": 8080}) + with pytest.raises(Exception): + render.portals.add_portal({"scheme": "http", "path": "/", "port": 8080}) + + +def test_add_duplicate_portal_with_explicit_name(mock_values): + render = Render(mock_values) + render.portals.add_portal({"name": "Some Portal", "scheme": "http", "path": "/", "port": 8080}) + with pytest.raises(Exception): + render.portals.add_portal({"name": "Some Portal", "scheme": "http", "path": "/", "port": 8080}) + + +def test_add_portal_with_invalid_scheme(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.portals.add_portal({"scheme": "invalid_scheme", "path": "/", "port": 8080}) + + +def test_add_portal_with_invalid_path(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.portals.add_portal({"scheme": "http", "path": "invalid_path", "port": 8080}) + + +def test_add_portal_with_invalid_path_double_slash(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.portals.add_portal({"scheme": "http", "path": "/some//path", "port": 8080}) + + +def test_add_portal_with_invalid_port(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.portals.add_portal({"scheme": "http", "path": "/", "port": -1}) diff --git a/library/2.0.34/tests/test_ports.py b/library/2.0.34/tests/test_ports.py new file mode 100644 index 0000000000..a4c923ca1d --- /dev/null +++ b/library/2.0.34/tests/test_ports.py @@ -0,0 +1,110 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_ports(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.ports.add_port(8081, 8080) + c1.ports.add_port(8082, 8080, {"protocol": "udp"}) + output = render.render() + assert output["services"]["test_container"]["ports"] == [ + {"published": 8081, "target": 8080, "protocol": "tcp", "mode": "ingress", "host_ip": "0.0.0.0"}, + {"published": 8082, "target": 8080, "protocol": "udp", "mode": "ingress", "host_ip": "0.0.0.0"}, + ] + + +def test_add_duplicate_ports(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.ports.add_port(8081, 8080) + c1.ports.add_port(8081, 8080, {"protocol": "udp"}) # This should not raise + with pytest.raises(Exception): + c1.ports.add_port(8081, 8080) + + +def test_add_duplicate_ports_with_different_host_ip(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.ports.add_port(8081, 8080, {"host_ip": "192.168.1.10"}) + c1.ports.add_port(8081, 8080, {"host_ip": "192.168.1.11"}) + output = render.render() + assert output["services"]["test_container"]["ports"] == [ + {"published": 8081, "target": 8080, "protocol": "tcp", "mode": "ingress", "host_ip": "192.168.1.10"}, + {"published": 8081, "target": 8080, "protocol": "tcp", "mode": "ingress", "host_ip": "192.168.1.11"}, + ] + + +def test_add_duplicate_ports_to_specific_host_ip_binds_to_0_0_0_0(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.ports.add_port(8081, 8080, {"host_ip": "192.168.1.10"}) + with pytest.raises(Exception): + c1.ports.add_port(8081, 8080, {"host_ip": "0.0.0.0"}) + + +def test_add_duplicate_ports_to_0_0_0_0_binds_to_specific_host_ip(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.ports.add_port(8081, 8080, {"host_ip": "0.0.0.0"}) + with pytest.raises(Exception): + c1.ports.add_port(8081, 8080, {"host_ip": "192.168.1.10"}) + + +def test_add_ports_with_invalid_protocol(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.ports.add_port(8081, 8080, {"protocol": "invalid_protocol"}) + + +def test_add_ports_with_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.ports.add_port(8081, 8080, {"mode": "invalid_mode"}) + + +def test_add_ports_with_invalid_ip(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.ports.add_port(8081, 8080, {"host_ip": "invalid_ip"}) + + +def test_add_ports_with_invalid_host_port(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.ports.add_port(-1, 8080) + + +def test_add_ports_with_invalid_container_port(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.ports.add_port(8081, -1) diff --git a/library/2.0.34/tests/test_render.py b/library/2.0.34/tests/test_render.py new file mode 100644 index 0000000000..60dc00679e --- /dev/null +++ b/library/2.0.34/tests/test_render.py @@ -0,0 +1,37 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_values_cannot_be_modified(mock_values): + render = Render(mock_values) + render.values["test"] = "test" + with pytest.raises(Exception): + render.render() + + +def test_duplicate_containers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_no_containers(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.render() diff --git a/library/2.0.34/tests/test_resources.py b/library/2.0.34/tests/test_resources.py new file mode 100644 index 0000000000..cd83d164e5 --- /dev/null +++ b/library/2.0.34/tests/test_resources.py @@ -0,0 +1,140 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_automatically_add_cpu(mock_values): + mock_values["resources"] = {"limits": {"cpus": 1.0}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["deploy"]["resources"]["limits"]["cpus"] == "1.0" + + +def test_invalid_cpu(mock_values): + mock_values["resources"] = {"limits": {"cpus": "invalid"}} + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_automatically_add_memory(mock_values): + mock_values["resources"] = {"limits": {"memory": 1024}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["deploy"]["resources"]["limits"]["memory"] == "1024M" + + +def test_invalid_memory(mock_values): + mock_values["resources"] = {"limits": {"memory": "invalid"}} + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_automatically_add_gpus(mock_values): + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + devices = output["services"]["test_container"]["deploy"]["resources"]["reservations"]["devices"] + assert len(devices) == 1 + assert devices[0] == { + "capabilities": ["gpu"], + "driver": "nvidia", + "device_ids": ["uuid_0", "uuid_1"], + } + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_gpu_without_uuid(mock_values): + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_remove_cpus_and_memory_with_gpus(mock_values): + mock_values["resources"] = {"gpus": {"nvidia_gpu_selection": {"pci_slot_0": {"uuid": "uuid_1", "use_gpu": True}}}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.deploy.resources.remove_cpus_and_memory() + output = render.render() + assert "limits" not in output["services"]["test_container"]["deploy"]["resources"] + devices = output["services"]["test_container"]["deploy"]["resources"]["reservations"]["devices"] + assert len(devices) == 1 + assert devices[0] == { + "capabilities": ["gpu"], + "driver": "nvidia", + "device_ids": ["uuid_1"], + } + + +def test_remove_cpus_and_memory(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.deploy.resources.remove_cpus_and_memory() + output = render.render() + assert "deploy" not in output["services"]["test_container"] + + +def test_remove_devices(mock_values): + mock_values["resources"] = {"gpus": {"nvidia_gpu_selection": {"pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}}}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.deploy.resources.remove_devices() + output = render.render() + assert "reservations" not in output["services"]["test_container"]["deploy"]["resources"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_set_profile(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.deploy.resources.set_profile("low") + output = render.render() + assert output["services"]["test_container"]["deploy"]["resources"]["limits"]["cpus"] == "1" + assert output["services"]["test_container"]["deploy"]["resources"]["limits"]["memory"] == "512M" + + +def test_set_profile_invalid_profile(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.deploy.resources.set_profile("invalid_profile") diff --git a/library/2.0.34/tests/test_restart.py b/library/2.0.34/tests/test_restart.py new file mode 100644 index 0000000000..06b2975590 --- /dev/null +++ b/library/2.0.34/tests/test_restart.py @@ -0,0 +1,57 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_invalid_restart_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.restart.set_policy("invalid_policy") + + +def test_valid_restart_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.restart.set_policy("on-failure") + output = render.render() + assert output["services"]["test_container"]["restart"] == "on-failure" + + +def test_valid_restart_policy_with_maximum_retry_count(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.restart.set_policy("on-failure", 10) + output = render.render() + assert output["services"]["test_container"]["restart"] == "on-failure:10" + + +def test_invalid_restart_policy_with_maximum_retry_count(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.restart.set_policy("on-failure", maximum_retry_count=-1) + + +def test_invalid_restart_policy_with_maximum_retry_count_and_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.restart.set_policy("always", maximum_retry_count=10) diff --git a/library/2.0.34/tests/test_sysctls.py b/library/2.0.34/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/library/2.0.34/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/library/2.0.34/tests/test_volumes.py b/library/2.0.34/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/library/2.0.34/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/library/2.0.34/validations.py b/library/2.0.34/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/library/2.0.34/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/library/2.0.34/volume_mount.py b/library/2.0.34/volume_mount.py new file mode 100644 index 0000000000..aadd077750 --- /dev/null +++ b/library/2.0.34/volume_mount.py @@ -0,0 +1,92 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .formatter import merge_dicts_no_overwrite + from .volume_mount_types import BindMountType, VolumeMountType, TmpfsMountType + from .volume_sources import HostPathSource, IxVolumeSource, CifsSource, NfsSource, VolumeSource +except ImportError: + from error import RenderError + from formatter import merge_dicts_no_overwrite + from volume_mount_types import BindMountType, VolumeMountType, TmpfsMountType + from volume_sources import HostPathSource, IxVolumeSource, CifsSource, NfsSource, VolumeSource + + +class VolumeMount: + def __init__(self, render_instance: "Render", mount_path: str, config: "IxStorage"): + self._render_instance = render_instance + self.mount_path: str = mount_path + + storage_type: str = config.get("type", "") + if not storage_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match storage_type: + case "host_path": + spec_type = "bind" + mount_config = config.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + mount_type_specific_definition = BindMountType(self._render_instance, mount_config).render() + source = HostPathSource(self._render_instance, mount_config).get() + case "ix_volume": + spec_type = "bind" + mount_config = config.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + mount_type_specific_definition = BindMountType(self._render_instance, mount_config).render() + source = IxVolumeSource(self._render_instance, mount_config).get() + case "tmpfs": + spec_type = "tmpfs" + mount_config = config.get("tmpfs_config", {}) + mount_type_specific_definition = TmpfsMountType(self._render_instance, mount_config).render() + source = None + case "nfs": + spec_type = "volume" + mount_config = config.get("nfs_config") + if mount_config is None: + raise RenderError("Expected [nfs_config] to be set for [nfs] type.") + mount_type_specific_definition = VolumeMountType(self._render_instance, mount_config).render() + source = NfsSource(self._render_instance, mount_config).get() + case "cifs": + spec_type = "volume" + mount_config = config.get("cifs_config") + if mount_config is None: + raise RenderError("Expected [cifs_config] to be set for [cifs] type.") + mount_type_specific_definition = VolumeMountType(self._render_instance, mount_config).render() + source = CifsSource(self._render_instance, mount_config).get() + case "volume": + spec_type = "volume" + mount_config = config.get("volume_config") + if mount_config is None: + raise RenderError("Expected [volume_config] to be set for [volume] type.") + mount_type_specific_definition = VolumeMountType(self._render_instance, mount_config).render() + source = VolumeSource(self._render_instance, mount_config).get() + case "temporary": + spec_type = "volume" + mount_config = config.get("volume_config") + if mount_config is None: + raise RenderError("Expected [volume_config] to be set for [temporary] type.") + mount_type_specific_definition = VolumeMountType(self._render_instance, mount_config).render() + source = VolumeSource(self._render_instance, mount_config).get() + case "anonymous": + spec_type = "volume" + mount_config = config.get("volume_config") or {} + mount_type_specific_definition = VolumeMountType(self._render_instance, mount_config).render() + source = None + case _: + raise RenderError(f"Storage type [{storage_type}] is not supported for volume mounts.") + + common_spec = {"type": spec_type, "target": self.mount_path, "read_only": config.get("read_only", False)} + if source is not None: + common_spec["source"] = source + self._render_instance.volumes.add_volume(source, storage_type, mount_config) # type: ignore + + self.volume_mount_spec = merge_dicts_no_overwrite(common_spec, mount_type_specific_definition) + + def render(self) -> dict: + return self.volume_mount_spec diff --git a/library/2.0.34/volume_mount_types.py b/library/2.0.34/volume_mount_types.py new file mode 100644 index 0000000000..00a0ec3a18 --- /dev/null +++ b/library/2.0.34/volume_mount_types.py @@ -0,0 +1,72 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorageTmpfsConfig, IxStorageVolumeConfig, IxStorageBindLikeConfigs + + +try: + from .error import RenderError + from .validations import valid_host_path_propagation, valid_octal_mode_or_raise +except ImportError: + from error import RenderError + from validations import valid_host_path_propagation, valid_octal_mode_or_raise + + +class TmpfsMountType: + def __init__(self, render_instance: "Render", config: "IxStorageTmpfsConfig"): + self._render_instance = render_instance + self.spec = {"tmpfs": {}} + size = config.get("size", None) + mode = config.get("mode", None) + + if size is not None: + if not isinstance(size, int): + raise RenderError(f"Expected [size] to be an integer for [tmpfs] type, got [{size}]") + if not size > 0: + raise RenderError(f"Expected [size] to be greater than 0 for [tmpfs] type, got [{size}]") + # Convert Mebibytes to Bytes + self.spec["tmpfs"]["size"] = size * 1024 * 1024 + + if mode is not None: + mode = valid_octal_mode_or_raise(mode) + self.spec["tmpfs"]["mode"] = int(mode, 8) + + if not self.spec["tmpfs"]: + self.spec.pop("tmpfs") + + def render(self) -> dict: + """Render the tmpfs mount specification.""" + return self.spec + + +class BindMountType: + def __init__(self, render_instance: "Render", config: "IxStorageBindLikeConfigs"): + self._render_instance = render_instance + self.spec: dict = {} + + propagation = valid_host_path_propagation(config.get("propagation", "rprivate")) + create_host_path = config.get("create_host_path", False) + + self.spec: dict = { + "bind": { + "create_host_path": create_host_path, + "propagation": propagation, + } + } + + def render(self) -> dict: + """Render the bind mount specification.""" + return self.spec + + +class VolumeMountType: + def __init__(self, render_instance: "Render", config: "IxStorageVolumeConfig"): + self._render_instance = render_instance + self.spec: dict = {} + + self.spec: dict = {"volume": {"nocopy": config.get("nocopy", False)}} + + def render(self) -> dict: + """Render the volume mount specification.""" + return self.spec diff --git a/library/2.0.34/volume_sources.py b/library/2.0.34/volume_sources.py new file mode 100644 index 0000000000..c33fe55ea1 --- /dev/null +++ b/library/2.0.34/volume_sources.py @@ -0,0 +1,106 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorageHostPathConfig, IxStorageIxVolumeConfig, IxStorageVolumeConfig + +try: + from .error import RenderError + from .formatter import get_hashed_name_for_volume + from .validations import valid_fs_path_or_raise +except ImportError: + from error import RenderError + from formatter import get_hashed_name_for_volume + from validations import valid_fs_path_or_raise + + +class HostPathSource: + def __init__(self, render_instance: "Render", config: "IxStorageHostPathConfig"): + self._render_instance = render_instance + self.source: str = "" + + if not config: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + + path = "" + if config.get("acl_enable", False): + acl_path = config.get("acl", {}).get("path") + if not acl_path: + raise RenderError("Expected [host_path_config.acl.path] to be set for [host_path] type.") + path = valid_fs_path_or_raise(acl_path) + else: + path = valid_fs_path_or_raise(config.get("path", "")) + + self.source = path.rstrip("/") + + def get(self): + return self.source + + +class IxVolumeSource: + def __init__(self, render_instance: "Render", config: "IxStorageIxVolumeConfig"): + self._render_instance = render_instance + self.source: str = "" + + if not config: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + dataset_name = config.get("dataset_name") + if not dataset_name: + raise RenderError("Expected [ix_volume_config.dataset_name] to be set for [ix_volume] type.") + + ix_volumes = self._render_instance.values.get("ix_volumes", {}) + if dataset_name not in ix_volumes: + available = ", ".join(ix_volumes.keys()) + raise RenderError( + f"Expected the key [{dataset_name}] to be set in [ix_volumes] for [ix_volume] type. " + f"Available keys: [{available}]." + ) + + self.source = valid_fs_path_or_raise(ix_volumes[dataset_name].rstrip("/")) + + def get(self): + return self.source + + +class CifsSource: + def __init__(self, render_instance: "Render", config: dict): + self._render_instance = render_instance + self.source: str = "" + + if not config: + raise RenderError("Expected [cifs_config] to be set for [cifs] type.") + self.source = get_hashed_name_for_volume("cifs", config) + + def get(self): + return self.source + + +class NfsSource: + def __init__(self, render_instance: "Render", config: dict): + self._render_instance = render_instance + self.source: str = "" + + if not config: + raise RenderError("Expected [nfs_config] to be set for [nfs] type.") + self.source = get_hashed_name_for_volume("nfs", config) + + def get(self): + return self.source + + +class VolumeSource: + def __init__(self, render_instance: "Render", config: "IxStorageVolumeConfig"): + self._render_instance = render_instance + self.source: str = "" + + if not config: + raise RenderError("Expected [volume_config] to be set for [volume] type.") + + volume_name: str = config.get("volume_name", "") + if not volume_name: + raise RenderError("Expected [volume_config.volume_name] to be set for [volume] type.") + + self.source = volume_name + + def get(self): + return self.source diff --git a/library/2.0.34/volume_types.py b/library/2.0.34/volume_types.py new file mode 100644 index 0000000000..4ccea08f83 --- /dev/null +++ b/library/2.0.34/volume_types.py @@ -0,0 +1,133 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorageNfsConfig, IxStorageCifsConfig, IxStorageVolumeConfig + + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_fs_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_fs_path_or_raise + + +class NfsVolume: + def __init__(self, render_instance: "Render", config: "IxStorageNfsConfig"): + self._render_instance = render_instance + + if not config: + raise RenderError("Expected [nfs_config] to be set for [nfs] type") + + required_keys = ["server", "path"] + for key in required_keys: + if not config.get(key): + raise RenderError(f"Expected [{key}] to be set for [nfs] type") + + opts = [f"addr={config['server']}"] + cfg_options = config.get("options") + if cfg_options: + if not isinstance(cfg_options, list): + raise RenderError("Expected [nfs_config.options] to be a list for [nfs] type") + + tracked_keys: set[str] = set() + disallowed_opts = ["addr"] + for opt in cfg_options: + if not isinstance(opt, str): + raise RenderError("Options for [nfs] type must be a list of strings.") + + key = opt.split("=")[0] + if key in tracked_keys: + raise RenderError(f"Option [{key}] already added for [nfs] type.") + if key in disallowed_opts: + raise RenderError(f"Option [{key}] is not allowed for [nfs] type.") + opts.append(opt) + tracked_keys.add(key) + + opts.sort() + + path = valid_fs_path_or_raise(config["path"].rstrip("/")) + self.volume_spec = { + "driver_opts": { + "type": "nfs", + "device": f":{path}", + "o": f"{','.join([escape_dollar(opt) for opt in opts])}", + }, + } + + def get(self): + return self.volume_spec + + +class CifsVolume: + def __init__(self, render_instance: "Render", config: "IxStorageCifsConfig"): + self._render_instance = render_instance + self.volume_spec: dict = {} + + if not config: + raise RenderError("Expected [cifs_config] to be set for [cifs] type") + + required_keys = ["server", "path", "username", "password"] + for key in required_keys: + if not config.get(key): + raise RenderError(f"Expected [{key}] to be set for [cifs] type") + + opts = [ + "noperm", + f"user={config['username']}", + f"password={config['password']}", + ] + + domain = config.get("domain") + if domain: + opts.append(f"domain={domain}") + + cfg_options = config.get("options") + if cfg_options: + if not isinstance(cfg_options, list): + raise RenderError("Expected [cifs_config.options] to be a list for [cifs] type") + + tracked_keys: set[str] = set() + disallowed_opts = ["user", "password", "domain", "noperm"] + for opt in cfg_options: + if not isinstance(opt, str): + raise RenderError("Options for [cifs] type must be a list of strings.") + + key = opt.split("=")[0] + if key in tracked_keys: + raise RenderError(f"Option [{key}] already added for [cifs] type.") + if key in disallowed_opts: + raise RenderError(f"Option [{key}] is not allowed for [cifs] type.") + for disallowed in disallowed_opts: + if key == disallowed: + raise RenderError(f"Option [{key}] is not allowed for [cifs] type.") + opts.append(opt) + tracked_keys.add(key) + opts.sort() + + server = config["server"].lstrip("/") + path = config["path"].strip("/") + path = valid_fs_path_or_raise("/" + path).lstrip("/") + + self.volume_spec = { + "driver_opts": { + "type": "cifs", + "device": f"//{server}/{path}", + "o": f"{','.join([escape_dollar(opt) for opt in opts])}", + }, + } + + def get(self): + return self.volume_spec + + +class DockerVolume: + def __init__(self, render_instance: "Render", config: "IxStorageVolumeConfig"): + self._render_instance = render_instance + self.volume_spec: dict = {} + + def get(self): + return self.volume_spec diff --git a/library/2.0.34/volumes.py b/library/2.0.34/volumes.py new file mode 100644 index 0000000000..e6925a402f --- /dev/null +++ b/library/2.0.34/volumes.py @@ -0,0 +1,66 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + + +try: + from .error import RenderError + from .storage import IxStorageVolumeLikeConfigs + from .volume_types import NfsVolume, CifsVolume, DockerVolume +except ImportError: + from error import RenderError + from storage import IxStorageVolumeLikeConfigs + from volume_types import NfsVolume, CifsVolume, DockerVolume + + +class Volumes: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volumes: dict[str, Volume] = {} + + def add_volume( + self, + source: str, + storage_type: str, + config: "IxStorageVolumeLikeConfigs", + ): + # This method can be called many times from the volume mounts + # Only add the volume if it is not already added, but dont raise an error + if source == "": + raise RenderError(f"Volume source [{source}] cannot be empty") + + if source in self._volumes: + return + + self._volumes[source] = Volume(self._render_instance, storage_type, config) + + def has_volumes(self) -> bool: + return bool(self._volumes) + + def render(self): + return {name: v.render() for name, v in sorted(self._volumes.items()) if v.render() is not None} + + +class Volume: + def __init__( + self, + render_instance: "Render", + storage_type: str, + config: "IxStorageVolumeLikeConfigs", + ): + self._render_instance = render_instance + self.volume_spec: dict | None = {} + + match storage_type: + case "nfs": + self.volume_spec = NfsVolume(self._render_instance, config).get() # type: ignore + case "cifs": + self.volume_spec = CifsVolume(self._render_instance, config).get() # type: ignore + case "volume" | "temporary": + self.volume_spec = DockerVolume(self._render_instance, config).get() # type: ignore + case _: + self.volume_spec = None + + def render(self): + return self.volume_spec diff --git a/library/hashes.yaml b/library/hashes.yaml index a470c76c17..03b8f534f4 100644 --- a/library/hashes.yaml +++ b/library/hashes.yaml @@ -1,3 +1,3 @@ 0.0.1: f074617a82a86d2a6cc78a4c8a4296fc9d168e456f12713e50c696557b302133 1.1.7: d05e43e25b7dc1736be6cc1efa4b9255368aa346e3e7a4350a38440f29b73186 -2.0.33: 167a3532cd10bbb030a484b3b2a84248fad6d59e9ad0338ad0bb6c543dfca949 +2.0.34: 54c507c0a47e2a10aee421ec5a67c04f73692bd865d4d5057f57711e322a7951 From 9aee55691da0c3415e40a2e9ee992ade168d78d1 Mon Sep 17 00:00:00 2001 From: Stavros Kois <47820033+stavros-k@users.noreply.github.com> Date: Thu, 5 Dec 2024 09:39:30 +0200 Subject: [PATCH 19/19] rm old --- library/2.0.33/__init__.py | 0 library/2.0.33/configs.py | 86 --- library/2.0.33/container.py | 317 ----------- library/2.0.33/depends.py | 34 -- library/2.0.33/deploy.py | 24 - library/2.0.33/deps.py | 454 --------------- library/2.0.33/device.py | 31 -- library/2.0.33/devices.py | 68 --- library/2.0.33/dns.py | 79 --- library/2.0.33/environment.py | 109 ---- library/2.0.33/error.py | 4 - library/2.0.33/formatter.py | 26 - library/2.0.33/functions.py | 149 ----- library/2.0.33/healthcheck.py | 193 ------- library/2.0.33/labels.py | 37 -- library/2.0.33/notes.py | 70 --- library/2.0.33/portal.py | 22 - library/2.0.33/portals.py | 28 - library/2.0.33/ports.py | 68 --- library/2.0.33/render.py | 89 --- library/2.0.33/resources.py | 115 ---- library/2.0.33/restart.py | 25 - library/2.0.33/storage.py | 116 ---- library/2.0.33/sysctls.py | 38 -- library/2.0.33/tests/__init__.py | 0 library/2.0.33/tests/test_build_image.py | 49 -- library/2.0.33/tests/test_configs.py | 63 --- library/2.0.33/tests/test_container.py | 324 ----------- library/2.0.33/tests/test_depends.py | 54 -- library/2.0.33/tests/test_deps.py | 380 ------------- library/2.0.33/tests/test_device.py | 131 ----- library/2.0.33/tests/test_dns.py | 64 --- library/2.0.33/tests/test_environment.py | 184 ------- library/2.0.33/tests/test_formatter.py | 13 - library/2.0.33/tests/test_functions.py | 88 --- library/2.0.33/tests/test_healthcheck.py | 187 ------- library/2.0.33/tests/test_labels.py | 88 --- library/2.0.33/tests/test_notes.py | 213 -------- library/2.0.33/tests/test_portal.py | 75 --- library/2.0.33/tests/test_ports.py | 110 ---- library/2.0.33/tests/test_render.py | 37 -- library/2.0.33/tests/test_resources.py | 140 ----- library/2.0.33/tests/test_restart.py | 57 -- library/2.0.33/tests/test_sysctls.py | 62 --- library/2.0.33/tests/test_volumes.py | 666 ----------------------- library/2.0.33/validations.py | 227 -------- library/2.0.33/volume_mount.py | 92 ---- library/2.0.33/volume_mount_types.py | 72 --- library/2.0.33/volume_sources.py | 106 ---- library/2.0.33/volume_types.py | 133 ----- library/2.0.33/volumes.py | 66 --- 51 files changed, 5863 deletions(-) delete mode 100644 library/2.0.33/__init__.py delete mode 100644 library/2.0.33/configs.py delete mode 100644 library/2.0.33/container.py delete mode 100644 library/2.0.33/depends.py delete mode 100644 library/2.0.33/deploy.py delete mode 100644 library/2.0.33/deps.py delete mode 100644 library/2.0.33/device.py delete mode 100644 library/2.0.33/devices.py delete mode 100644 library/2.0.33/dns.py delete mode 100644 library/2.0.33/environment.py delete mode 100644 library/2.0.33/error.py delete mode 100644 library/2.0.33/formatter.py delete mode 100644 library/2.0.33/functions.py delete mode 100644 library/2.0.33/healthcheck.py delete mode 100644 library/2.0.33/labels.py delete mode 100644 library/2.0.33/notes.py delete mode 100644 library/2.0.33/portal.py delete mode 100644 library/2.0.33/portals.py delete mode 100644 library/2.0.33/ports.py delete mode 100644 library/2.0.33/render.py delete mode 100644 library/2.0.33/resources.py delete mode 100644 library/2.0.33/restart.py delete mode 100644 library/2.0.33/storage.py delete mode 100644 library/2.0.33/sysctls.py delete mode 100644 library/2.0.33/tests/__init__.py delete mode 100644 library/2.0.33/tests/test_build_image.py delete mode 100644 library/2.0.33/tests/test_configs.py delete mode 100644 library/2.0.33/tests/test_container.py delete mode 100644 library/2.0.33/tests/test_depends.py delete mode 100644 library/2.0.33/tests/test_deps.py delete mode 100644 library/2.0.33/tests/test_device.py delete mode 100644 library/2.0.33/tests/test_dns.py delete mode 100644 library/2.0.33/tests/test_environment.py delete mode 100644 library/2.0.33/tests/test_formatter.py delete mode 100644 library/2.0.33/tests/test_functions.py delete mode 100644 library/2.0.33/tests/test_healthcheck.py delete mode 100644 library/2.0.33/tests/test_labels.py delete mode 100644 library/2.0.33/tests/test_notes.py delete mode 100644 library/2.0.33/tests/test_portal.py delete mode 100644 library/2.0.33/tests/test_ports.py delete mode 100644 library/2.0.33/tests/test_render.py delete mode 100644 library/2.0.33/tests/test_resources.py delete mode 100644 library/2.0.33/tests/test_restart.py delete mode 100644 library/2.0.33/tests/test_sysctls.py delete mode 100644 library/2.0.33/tests/test_volumes.py delete mode 100644 library/2.0.33/validations.py delete mode 100644 library/2.0.33/volume_mount.py delete mode 100644 library/2.0.33/volume_mount_types.py delete mode 100644 library/2.0.33/volume_sources.py delete mode 100644 library/2.0.33/volume_types.py delete mode 100644 library/2.0.33/volumes.py diff --git a/library/2.0.33/__init__.py b/library/2.0.33/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/library/2.0.33/configs.py b/library/2.0.33/configs.py deleted file mode 100644 index b76f4b169c..0000000000 --- a/library/2.0.33/configs.py +++ /dev/null @@ -1,86 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise - - -class Configs: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._configs: dict[str, dict] = {} - - def add(self, name: str, data: str): - if not isinstance(data, str): - raise RenderError(f"Expected [data] to be a string, got [{type(data)}]") - - if name not in self._configs: - self._configs[name] = {"name": name, "data": data} - return - - if data == self._configs[name]["data"]: - return - - raise RenderError(f"Config [{name}] already added with different data") - - def has_configs(self): - return bool(self._configs) - - def render(self): - return { - c["name"]: {"content": escape_dollar(c["data"])} - for c in sorted(self._configs.values(), key=lambda c: c["name"]) - } - - -class ContainerConfigs: - def __init__(self, render_instance: "Render", configs: Configs): - self._render_instance = render_instance - self.top_level_configs: Configs = configs - self.container_configs: set[ContainerConfig] = set() - - def add(self, name: str, data: str, target: str, mode: str = ""): - self.top_level_configs.add(name, data) - - if target == "": - raise RenderError(f"Expected [target] to be set for config [{name}]") - if mode != "": - mode = valid_octal_mode_or_raise(mode) - - if target in [c.target for c in self.container_configs]: - raise RenderError(f"Target [{target}] already used for another config") - target = valid_fs_path_or_raise(target) - self.container_configs.add(ContainerConfig(self._render_instance, name, target, mode)) - - def has_configs(self): - return bool(self.container_configs) - - def render(self): - return [c.render() for c in sorted(self.container_configs, key=lambda c: c.source)] - - -class ContainerConfig: - def __init__(self, render_instance: "Render", source: str, target: str, mode: str): - self._render_instance = render_instance - self.source = source - self.target = target - self.mode = mode - - def render(self): - result: dict[str, str | int] = { - "source": self.source, - "target": self.target, - } - - if self.mode: - result["mode"] = int(self.mode, 8) - - return result diff --git a/library/2.0.33/container.py b/library/2.0.33/container.py deleted file mode 100644 index a95e76734c..0000000000 --- a/library/2.0.33/container.py +++ /dev/null @@ -1,317 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage - from .sysctls import Sysctls -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - from sysctls import Sysctls - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.sysctls: Sysctls = Sysctls(self._render_instance, self) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - self.add_group(999) - self._storage._add_docker_socket(read_only, mount_path) - - def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): - self._storage._add_tun_device(read_only, mount_path) - - def add_snd_device(self): - self.add_group(29) - self.devices._add_snd_device() - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = f"{self._grace_period}s" - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_gpus(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self.sysctls.has_sysctls(): - result["sysctls"] = self.sysctls.render() - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/library/2.0.33/depends.py b/library/2.0.33/depends.py deleted file mode 100644 index 4e057cf085..0000000000 --- a/library/2.0.33/depends.py +++ /dev/null @@ -1,34 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_depend_condition_or_raise -except ImportError: - from error import RenderError - from validations import valid_depend_condition_or_raise - - -class Depends: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._dependencies: dict[str, str] = {} - - def add_dependency(self, name: str, condition: str): - condition = valid_depend_condition_or_raise(condition) - if name in self._dependencies.keys(): - raise RenderError(f"Dependency [{name}] already added") - if name not in self._render_instance.container_names(): - raise RenderError( - f"Dependency [{name}] not found in defined containers. " - f"Available containers: [{', '.join(self._render_instance.container_names())}]" - ) - self._dependencies[name] = condition - - def has_dependencies(self): - return len(self._dependencies) > 0 - - def render(self): - return {d: {"condition": c} for d, c in self._dependencies.items()} diff --git a/library/2.0.33/deploy.py b/library/2.0.33/deploy.py deleted file mode 100644 index 894dbc643b..0000000000 --- a/library/2.0.33/deploy.py +++ /dev/null @@ -1,24 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .resources import Resources -except ImportError: - from resources import Resources - - -class Deploy: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self.resources: Resources = Resources(self._render_instance) - - def has_deploy(self): - return self.resources.has_resources() - - def render(self): - if self.resources.has_resources(): - return {"resources": self.resources.render()} - - return {} diff --git a/library/2.0.33/deps.py b/library/2.0.33/deps.py deleted file mode 100644 index b3607fa6ab..0000000000 --- a/library/2.0.33/deps.py +++ /dev/null @@ -1,454 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - recursive = action_config.get("recursive", False) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - recursive = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "recursive": recursive, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod, recursive=False): - print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chmod(os.path.join(root, f), int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid, recursive=False): - print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chown(os.path.join(root, f), uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/library/2.0.33/device.py b/library/2.0.33/device.py deleted file mode 100644 index bfe97097cb..0000000000 --- a/library/2.0.33/device.py +++ /dev/null @@ -1,31 +0,0 @@ -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise, allowed_device_or_raise, valid_cgroup_perm_or_raise -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise, allowed_device_or_raise, valid_cgroup_perm_or_raise - - -class Device: - def __init__(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - hd = valid_fs_path_or_raise(host_device.rstrip("/")) - cd = valid_fs_path_or_raise(container_device.rstrip("/")) - if not hd or not cd: - raise RenderError( - "Expected [host_device] and [container_device] to be set. " - f"Got host_device [{host_device}] and container_device [{container_device}]" - ) - - cgroup_perm = valid_cgroup_perm_or_raise(cgroup_perm) - if not allow_disallowed: - hd = allowed_device_or_raise(hd) - - self.cgroup_perm: str = cgroup_perm - self.host_device: str = hd - self.container_device: str = cd - - def render(self): - result = f"{self.host_device}:{self.container_device}" - if self.cgroup_perm: - result += f":{self.cgroup_perm}" - return result diff --git a/library/2.0.33/devices.py b/library/2.0.33/devices.py deleted file mode 100644 index b6139371ee..0000000000 --- a/library/2.0.33/devices.py +++ /dev/null @@ -1,68 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - if resources["gpus"].get("kfd_device_exists", False): - self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def _add_snd_device(self): - self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/library/2.0.33/dns.py b/library/2.0.33/dns.py deleted file mode 100644 index d3ae7b19fa..0000000000 --- a/library/2.0.33/dns.py +++ /dev/null @@ -1,79 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import allowed_dns_opt_or_raise -except ImportError: - from error import RenderError - from validations import allowed_dns_opt_or_raise - - -class Dns: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._dns_options: set[str] = set() - self._dns_searches: set[str] = set() - self._dns_nameservers: set[str] = set() - - self._auto_add_dns_opts_from_values() - self._auto_add_dns_searches_from_values() - self._auto_add_dns_nameservers_from_values() - - def _get_dns_opt_keys(self): - return [self._get_key_from_opt(opt) for opt in self._dns_options] - - def _get_key_from_opt(self, opt): - return opt.split(":")[0] - - def _auto_add_dns_opts_from_values(self): - values = self._render_instance.values - for dns_opt in values.get("network", {}).get("dns_opts", []): - self.add_dns_opt(dns_opt) - - def _auto_add_dns_searches_from_values(self): - values = self._render_instance.values - for dns_search in values.get("network", {}).get("dns_searches", []): - self.add_dns_search(dns_search) - - def _auto_add_dns_nameservers_from_values(self): - values = self._render_instance.values - for dns_nameserver in values.get("network", {}).get("dns_nameservers", []): - self.add_dns_nameserver(dns_nameserver) - - def add_dns_search(self, dns_search): - if dns_search in self._dns_searches: - raise RenderError(f"DNS Search [{dns_search}] already added") - self._dns_searches.add(dns_search) - - def add_dns_nameserver(self, dns_nameserver): - if dns_nameserver in self._dns_nameservers: - raise RenderError(f"DNS Nameserver [{dns_nameserver}] already added") - self._dns_nameservers.add(dns_nameserver) - - def add_dns_opt(self, dns_opt): - # eg attempts:3 - key = allowed_dns_opt_or_raise(self._get_key_from_opt(dns_opt)) - if key in self._get_dns_opt_keys(): - raise RenderError(f"DNS Option [{key}] already added") - self._dns_options.add(dns_opt) - - def has_dns_opts(self): - return len(self._dns_options) > 0 - - def has_dns_searches(self): - return len(self._dns_searches) > 0 - - def has_dns_nameservers(self): - return len(self._dns_nameservers) > 0 - - def render_dns_searches(self): - return sorted(self._dns_searches) - - def render_dns_opts(self): - return sorted(self._dns_options) - - def render_dns_nameservers(self): - return sorted(self._dns_nameservers) diff --git a/library/2.0.33/environment.py b/library/2.0.33/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/library/2.0.33/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/library/2.0.33/error.py b/library/2.0.33/error.py deleted file mode 100644 index aef48d3b02..0000000000 --- a/library/2.0.33/error.py +++ /dev/null @@ -1,4 +0,0 @@ -class RenderError(Exception): - """Base class for exceptions in this module.""" - - pass diff --git a/library/2.0.33/formatter.py b/library/2.0.33/formatter.py deleted file mode 100644 index 24e882f47a..0000000000 --- a/library/2.0.33/formatter.py +++ /dev/null @@ -1,26 +0,0 @@ -import json -import hashlib - - -def escape_dollar(text: str) -> str: - return text.replace("$", "$$") - - -def get_hashed_name_for_volume(prefix: str, config: dict): - config_hash = hashlib.sha256(json.dumps(config).encode("utf-8")).hexdigest() - return f"{prefix}_{config_hash}" - - -def get_hash_with_prefix(prefix: str, data: str): - return f"{prefix}_{hashlib.sha256(data.encode('utf-8')).hexdigest()}" - - -def merge_dicts_no_overwrite(dict1, dict2): - overlapping_keys = dict1.keys() & dict2.keys() - if overlapping_keys: - raise ValueError(f"Merging of dicts failed. Overlapping keys: {overlapping_keys}") - return {**dict1, **dict2} - - -def get_image_with_hashed_data(image: str, data: str): - return get_hash_with_prefix(f"ix-{image}", data) diff --git a/library/2.0.33/functions.py b/library/2.0.33/functions.py deleted file mode 100644 index 7d082d8c46..0000000000 --- a/library/2.0.33/functions.py +++ /dev/null @@ -1,149 +0,0 @@ -import re -import copy -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .volume_sources import HostPathSource, IxVolumeSource -except ImportError: - from error import RenderError - from volume_sources import HostPathSource, IxVolumeSource - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return copy.deepcopy(dict) - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string: str, chars: list[str], key: str): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - return string - - def _or_default(self, value, default): - if not value: - return default - return value - - def _temp_config(self, name): - if not name: - raise RenderError("Expected [name] to be set when calling [temp_config].") - return {"type": "temporary", "volume_config": {"volume_name": name}} - - def _get_host_path(self, storage): - source_type = storage.get("type", "") - if not source_type: - raise RenderError("Expected [type] to be set for volume mounts.") - - match source_type: - case "host_path": - mount_config = storage.get("host_path_config") - if mount_config is None: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - host_source = HostPathSource(self._render_instance, mount_config).get() - return host_source - case "ix_volume": - mount_config = storage.get("ix_volume_config") - if mount_config is None: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - ix_source = IxVolumeSource(self._render_instance, mount_config).get() - return ix_source - case _: - raise RenderError(f"Storage type [{source_type}] does not support host path.") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - "get_host_path": self._get_host_path, - "or_default": self._or_default, - "temp_config": self._temp_config, - } diff --git a/library/2.0.33/healthcheck.py b/library/2.0.33/healthcheck.py deleted file mode 100644 index 36ae5d90aa..0000000000 --- a/library/2.0.33/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/library/2.0.33/labels.py b/library/2.0.33/labels.py deleted file mode 100644 index f1e667ba00..0000000000 --- a/library/2.0.33/labels.py +++ /dev/null @@ -1,37 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar -except ImportError: - from error import RenderError - from formatter import escape_dollar - - -class Labels: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._labels: dict[str, str] = {} - - def add_label(self, key: str, value: str): - if not key: - raise RenderError("Labels must have a key") - - if key.startswith("com.docker.compose"): - raise RenderError(f"Label [{key}] cannot start with [com.docker.compose] as it is reserved") - - if key in self._labels.keys(): - raise RenderError(f"Label [{key}] already added") - - self._labels[key] = escape_dollar(str(value)) - - def has_labels(self) -> bool: - return bool(self._labels) - - def render(self) -> dict[str, str]: - if not self.has_labels(): - return {} - return {label: value for label, value in sorted(self._labels.items())} diff --git a/library/2.0.33/notes.py b/library/2.0.33/notes.py deleted file mode 100644 index 4adc50c3d8..0000000000 --- a/library/2.0.33/notes.py +++ /dev/null @@ -1,70 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - - -class Notes: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._app_name: str = "" - self._warnings: list[str] = [] - self._deprecations: list[str] = [] - self._header: str = "" - self._body: str = "" - self._footer: str = "" - - self._auto_set_app_name() - self._auto_set_header() - self._auto_set_footer() - - def _auto_set_app_name(self): - app_name = self._render_instance.values.get("ix_context", {}).get("app_metadata", {}).get("name", "") - self._app_name = app_name or "" - - def _auto_set_header(self): - head = "# Welcome to TrueNAS SCALE\n\n" - head += f"Thank you for installing {self._app_name}!\n\n" - self._header = head - - def _auto_set_footer(self): - footer = "## Documentation\n\n" - footer += f"Documentation for {self._app_name} can be found at https://www.truenas.com/docs.\n\n" - footer += "## Bug reports\n\n" - footer += "If you find a bug in this app, please file an issue at\n" - footer += "https://ixsystems.atlassian.net or https://github.com/truenas/apps\n\n" - footer += "## Feature requests or improvements\n\n" - footer += "If you find a feature request for this app, please file an issue at\n" - footer += "https://ixsystems.atlassian.net or https://github.com/truenas/apps\n" - self._footer = footer - - def add_warning(self, warning: str): - self._warnings.append(warning) - - def add_deprecation(self, deprecation: str): - self._deprecations.append(deprecation) - - def set_body(self, body: str): - self._body = body - - def render(self): - result = self._header - - if self._warnings: - result += "## Warnings\n\n" - for warning in self._warnings: - result += f"- {warning}\n" - result += "\n" - - if self._deprecations: - result += "## Deprecations\n\n" - for deprecation in self._deprecations: - result += f"- {deprecation}\n" - result += "\n" - - if self._body: - result += self._body.strip() + "\n\n" - - result += self._footer - - return result diff --git a/library/2.0.33/portal.py b/library/2.0.33/portal.py deleted file mode 100644 index cf47163439..0000000000 --- a/library/2.0.33/portal.py +++ /dev/null @@ -1,22 +0,0 @@ -try: - from .validations import valid_portal_scheme_or_raise, valid_http_path_or_raise, valid_port_or_raise -except ImportError: - from validations import valid_portal_scheme_or_raise, valid_http_path_or_raise, valid_port_or_raise - - -class Portal: - def __init__(self, name: str, config: dict): - self._name = name - self._scheme = valid_portal_scheme_or_raise(config.get("scheme", "http")) - self._host = config.get("host", "0.0.0.0") or "0.0.0.0" - self._port = valid_port_or_raise(config.get("port", 0)) - self._path = valid_http_path_or_raise(config.get("path", "/")) - - def render(self): - return { - "name": self._name, - "scheme": self._scheme, - "host": self._host, - "port": self._port, - "path": self._path, - } diff --git a/library/2.0.33/portals.py b/library/2.0.33/portals.py deleted file mode 100644 index e106d231e6..0000000000 --- a/library/2.0.33/portals.py +++ /dev/null @@ -1,28 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .portal import Portal -except ImportError: - from error import RenderError - from portal import Portal - - -class Portals: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._portals: set[Portal] = set() - - def add_portal(self, config: dict): - name = config.get("name", "Web UI") - - if name in [p._name for p in self._portals]: - raise RenderError(f"Portal [{name}] already added") - - self._portals.add(Portal(name, config)) - - def render(self): - return [p.render() for _, p in sorted([(p._name, p) for p in self._portals])] diff --git a/library/2.0.33/ports.py b/library/2.0.33/ports.py deleted file mode 100644 index f11e1481b4..0000000000 --- a/library/2.0.33/ports.py +++ /dev/null @@ -1,68 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_port_protocol_or_raise, - valid_port_mode_or_raise, - valid_ip_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_port_protocol_or_raise, - valid_port_mode_or_raise, - valid_ip_or_raise, - ) - - -class Ports: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._ports: dict[str, dict] = {} - - def add_port(self, host_port: int, container_port: int, config: dict | None = None): - config = config or {} - host_port = valid_port_or_raise(host_port) - container_port = valid_port_or_raise(container_port) - proto = valid_port_protocol_or_raise(config.get("protocol", "tcp")) - mode = valid_port_mode_or_raise(config.get("mode", "ingress")) - host_ip = valid_ip_or_raise(config.get("host_ip", "0.0.0.0")) - - key = f"{host_port}_{host_ip}_{proto}" - if key in self._ports.keys(): - raise RenderError(f"Port [{host_port}/{proto}] already added for [{host_ip}]") - - if host_ip != "0.0.0.0": - # If the port we are adding is not going to use 0.0.0.0 - # Make sure that we don't have already added that port/proto to 0.0.0.0 - search_key = f"{host_port}_0.0.0.0_{proto}" - if search_key in self._ports.keys(): - raise RenderError(f"Cannot bind port [{host_port}/{proto}] to [{host_ip}], already bound to [0.0.0.0]") - elif host_ip == "0.0.0.0": - # If the port we are adding is going to use 0.0.0.0 - # Make sure that we don't have already added that port/proto to a specific ip - for p in self._ports.values(): - if p["published"] == host_port and p["protocol"] == proto: - raise RenderError( - f"Cannot bind port [{host_port}/{proto}] to [{host_ip}], already bound to [{p['host_ip']}]" - ) - - self._ports[key] = { - "published": host_port, - "target": container_port, - "protocol": proto, - "mode": mode, - "host_ip": host_ip, - } - - def has_ports(self): - return len(self._ports) > 0 - - def render(self): - return [config for _, config in sorted(self._ports.items())] diff --git a/library/2.0.33/render.py b/library/2.0.33/render.py deleted file mode 100644 index 9d8fcc28d5..0000000000 --- a/library/2.0.33/render.py +++ /dev/null @@ -1,89 +0,0 @@ -import copy - -try: - from .configs import Configs - from .container import Container - from .deps import Deps - from .error import RenderError - from .functions import Functions - from .notes import Notes - from .portals import Portals - from .volumes import Volumes -except ImportError: - from configs import Configs - from container import Container - from deps import Deps - from error import RenderError - from functions import Functions - from notes import Notes - from portals import Portals - from volumes import Volumes - - -class Render(object): - def __init__(self, values): - self._containers: dict[str, Container] = {} - self.values = values - self._add_images_internal_use() - # Make a copy after we inject the images - self._original_values: dict = copy.deepcopy(self.values) - - self.deps: Deps = Deps(self) - - self.configs = Configs(render_instance=self) - self.funcs = Functions(render_instance=self).func_map() - self.portals: Portals = Portals(render_instance=self) - self.notes: Notes = Notes(render_instance=self) - self.volumes = Volumes(render_instance=self) - - def _add_images_internal_use(self): - if not self.values.get("images"): - self.values["images"] = {} - - if "python_permissions_image" not in self.values["images"]: - self.values["images"]["python_permissions_image"] = {"repository": "python", "tag": "3.13.0-slim-bookworm"} - - def container_names(self): - return list(self._containers.keys()) - - def add_container(self, name: str, image: str): - name = name.strip() - if not name: - raise RenderError("Container name cannot be empty") - container = Container(self, name, image) - if name in self._containers: - raise RenderError(f"Container {name} already exists.") - self._containers[name] = container - return container - - def render(self): - if self.values != self._original_values: - raise RenderError("Values have been modified since the renderer was created.") - - if not self._containers: - raise RenderError("No containers added.") - - result: dict = { - "x-notes": self.notes.render(), - "x-portals": self.portals.render(), - "services": {c._name: c.render() for c in self._containers.values()}, - } - - # Make sure that after services are rendered - # there are no labels that target a non-existent container - # This is to prevent typos - for label in self.values.get("labels", []): - for c in label.get("containers", []): - if c not in self.container_names(): - raise RenderError(f"Label [{label['key']}] references container [{c}] which does not exist") - - if self.volumes.has_volumes(): - result["volumes"] = self.volumes.render() - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - # if self.networks: - # result["networks"] = {...} - - return result diff --git a/library/2.0.33/resources.py b/library/2.0.33/resources.py deleted file mode 100644 index 733f43bb6f..0000000000 --- a/library/2.0.33/resources.py +++ /dev/null @@ -1,115 +0,0 @@ -import re -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -DEFAULT_CPUS = 2.0 -DEFAULT_MEMORY = 4096 - - -class Resources: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._limits: dict = {} - self._reservations: dict = {} - self._nvidia_ids: set[str] = set() - self._auto_add_cpu_from_values() - self._auto_add_memory_from_values() - self._auto_add_gpus_from_values() - - def _set_cpu(self, cpus: Any): - c = str(cpus) - if not re.match(r"^[1-9][0-9]*(\.[0-9]+)?$", c): - raise RenderError(f"Expected cpus to be a number or a float (minimum 1.0), got [{cpus}]") - self._limits.update({"cpus": c}) - - def _set_memory(self, memory: Any): - m = str(memory) - if not re.match(r"^[1-9][0-9]*$", m): - raise RenderError(f"Expected memory to be a number, got [{memory}]") - self._limits.update({"memory": f"{m}M"}) - - def _auto_add_cpu_from_values(self): - resources = self._render_instance.values.get("resources", {}) - self._set_cpu(resources.get("limits", {}).get("cpus", DEFAULT_CPUS)) - - def _auto_add_memory_from_values(self): - resources = self._render_instance.values.get("resources", {}) - self._set_memory(resources.get("limits", {}).get("memory", DEFAULT_MEMORY)) - - def _auto_add_gpus_from_values(self): - resources = self._render_instance.values.get("resources", {}) - gpus = resources.get("gpus", {}).get("nvidia_gpu_selection", {}) - if not gpus: - return - - for pci, gpu in gpus.items(): - if gpu.get("use_gpu", False): - if not gpu.get("uuid"): - raise RenderError(f"Expected [uuid] to be set for GPU in slot [{pci}] in [nvidia_gpu_selection]") - self._nvidia_ids.add(gpu["uuid"]) - - if self._nvidia_ids: - if not self._reservations: - self._reservations["devices"] = [] - self._reservations["devices"].append( - { - "capabilities": ["gpu"], - "driver": "nvidia", - "device_ids": sorted(self._nvidia_ids), - } - ) - - # This is only used on ix-app that we allow - # disabling cpus and memory. GPUs are only added - # if the user has requested them. - def remove_cpus_and_memory(self): - self._limits.pop("cpus", None) - self._limits.pop("memory", None) - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._reservations.pop("devices", None) - - def set_profile(self, profile: str): - cpu, memory = profile_mapping(profile) - self._set_cpu(cpu) - self._set_memory(memory) - - def has_resources(self): - return len(self._limits) > 0 or len(self._reservations) > 0 - - def has_gpus(self): - gpu_devices = [d for d in self._reservations.get("devices", []) if "gpu" in d["capabilities"]] - return len(gpu_devices) > 0 - - def render(self): - result = {} - if self._limits: - result["limits"] = self._limits - if self._reservations: - result["reservations"] = self._reservations - - return result - - -def profile_mapping(profile: str): - profiles = { - "low": (1, 512), - "medium": (2, 1024), - } - - if profile not in profiles: - raise RenderError( - f"Resource profile [{profile}] is not valid. Valid options are: [{', '.join(profiles.keys())}]" - ) - - return profiles[profile] diff --git a/library/2.0.33/restart.py b/library/2.0.33/restart.py deleted file mode 100644 index 2f6281af48..0000000000 --- a/library/2.0.33/restart.py +++ /dev/null @@ -1,25 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .validations import valid_restart_policy_or_raise -except ImportError: - from validations import valid_restart_policy_or_raise - - -class RestartPolicy: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._policy: str = "unless-stopped" - self._maximum_retry_count: int = 0 - - def set_policy(self, policy: str, maximum_retry_count: int = 0): - self._policy = valid_restart_policy_or_raise(policy, maximum_retry_count) - self._maximum_retry_count = maximum_retry_count - - def render(self): - if self._policy == "on-failure" and self._maximum_retry_count > 0: - return f"{self._policy}:{self._maximum_retry_count}" - return self._policy diff --git a/library/2.0.33/storage.py b/library/2.0.33/storage.py deleted file mode 100644 index e697ba902a..0000000000 --- a/library/2.0.33/storage.py +++ /dev/null @@ -1,116 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): - mount_path = valid_fs_path_or_raise(mount_path) - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): - mount_path = valid_fs_path_or_raise(mount_path) - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/library/2.0.33/sysctls.py b/library/2.0.33/sysctls.py deleted file mode 100644 index e6b8469f3b..0000000000 --- a/library/2.0.33/sysctls.py +++ /dev/null @@ -1,38 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from container import Container - -try: - from .error import RenderError - from .validations import valid_sysctl_or_raise -except ImportError: - from error import RenderError - from validations import valid_sysctl_or_raise - - -class Sysctls: - def __init__(self, render_instance: "Render", container_instance: "Container"): - self._render_instance = render_instance - self._container_instance = container_instance - self._sysctls: dict = {} - - def add(self, key: str, value): - key = key.strip() - if not key: - raise RenderError("Sysctls key cannot be empty") - if value is None: - raise RenderError(f"Sysctl [{key}] requires a value") - if key in self._sysctls: - raise RenderError(f"Sysctl [{key}] already added") - self._sysctls[key] = str(value) - - def has_sysctls(self): - return bool(self._sysctls) - - def render(self): - if not self.has_sysctls(): - return {} - host_net = self._container_instance._network_mode == "host" - return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/library/2.0.33/tests/__init__.py b/library/2.0.33/tests/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/library/2.0.33/tests/test_build_image.py b/library/2.0.33/tests/test_build_image.py deleted file mode 100644 index f30c1210ed..0000000000 --- a/library/2.0.33/tests/test_build_image.py +++ /dev/null @@ -1,49 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_build_image_with_from(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.build_image(["FROM test_image"]) - - -def test_build_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.build_image( - [ - "RUN echo hello", - None, - "", - "RUN echo world", - ] - ) - output = render.render() - assert ( - output["services"]["test_container"]["image"] - == "ix-nginx:latest_4a127145ea4c25511707e57005dd0ed457fe2f4932082c8f9faa339a450b6a99" - ) - assert output["services"]["test_container"]["build"] == { - "tags": ["ix-nginx:latest_4a127145ea4c25511707e57005dd0ed457fe2f4932082c8f9faa339a450b6a99"], - "dockerfile_inline": """FROM nginx:latest -RUN echo hello -RUN echo world -""", - } diff --git a/library/2.0.33/tests/test_configs.py b/library/2.0.33/tests/test_configs.py deleted file mode 100644 index 9049e473ea..0000000000 --- a/library/2.0.33/tests/test_configs.py +++ /dev/null @@ -1,63 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_duplicate_config_with_different_data(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.configs.add("test_config", "test_data", "/some/path") - with pytest.raises(Exception): - c1.configs.add("test_config", "test_data2", "/some/path") - - -def test_add_config_with_empty_target(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.configs.add("test_config", "test_data", "") - - -def test_add_duplicate_target(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.configs.add("test_config", "test_data", "/some/path") - with pytest.raises(Exception): - c1.configs.add("test_config2", "test_data2", "/some/path") - - -def test_add_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.configs.add("test_config", "$test_data", "/some/path") - output = render.render() - assert output["configs"]["test_config"]["content"] == "$$test_data" - assert output["services"]["test_container"]["configs"] == [{"source": "test_config", "target": "/some/path"}] - - -def test_add_config_with_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.configs.add("test_config", "test_data", "/some/path", "0777") - output = render.render() - assert output["configs"]["test_config"]["content"] == "test_data" - assert output["services"]["test_container"]["configs"] == [ - {"source": "test_config", "target": "/some/path", "mode": 511} - ] diff --git a/library/2.0.33/tests/test_container.py b/library/2.0.33/tests/test_container.py deleted file mode 100644 index 61a22a5df2..0000000000 --- a/library/2.0.33/tests/test_container.py +++ /dev/null @@ -1,324 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == "10s" - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 999] - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_tun_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_tun_device() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/dev/net/tun", - "target": "/dev/net/tun", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_snd_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_snd_device() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] - assert output["services"]["test_container"]["group_add"] == [29, 568] - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/library/2.0.33/tests/test_depends.py b/library/2.0.33/tests/test_depends.py deleted file mode 100644 index a1d8373927..0000000000 --- a/library/2.0.33/tests/test_depends.py +++ /dev/null @@ -1,54 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_dependency(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c2 = render.add_container("test_container2", "test_image") - c1.healthcheck.disable() - c2.healthcheck.disable() - c1.depends.add_dependency("test_container2", "service_started") - output = render.render() - assert output["services"]["test_container"]["depends_on"]["test_container2"] == {"condition": "service_started"} - - -def test_add_dependency_invalid_condition(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - render.add_container("test_container2", "test_image") - with pytest.raises(Exception): - c1.depends.add_dependency("test_container2", "invalid_condition") - - -def test_add_dependency_missing_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.depends.add_dependency("test_container2", "service_started") - - -def test_add_dependency_duplicate(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - render.add_container("test_container2", "test_image") - c1.depends.add_dependency("test_container2", "service_started") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.depends.add_dependency("test_container2", "service_started") diff --git a/library/2.0.33/tests/test_deps.py b/library/2.0.33/tests/test_deps.py deleted file mode 100644 index f9562ba4f2..0000000000 --- a/library/2.0.33/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/library/2.0.33/tests/test_device.py b/library/2.0.33/tests/test_device.py deleted file mode 100644 index c44437367d..0000000000 --- a/library/2.0.33/tests/test_device.py +++ /dev/null @@ -1,131 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_automatically_add_gpu_devices_and_kfd(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/library/2.0.33/tests/test_dns.py b/library/2.0.33/tests/test_dns.py deleted file mode 100644 index fe6b21e34f..0000000000 --- a/library/2.0.33/tests/test_dns.py +++ /dev/null @@ -1,64 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_dns_opts(mock_values): - mock_values["network"] = {"dns_opts": ["attempts:3", "opt1", "opt2"]} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["dns_opt"] == ["attempts:3", "opt1", "opt2"] - - -def test_auto_add_dns_searches(mock_values): - mock_values["network"] = {"dns_searches": ["search1", "search2"]} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["dns_search"] == ["search1", "search2"] - - -def test_auto_add_dns_nameservers(mock_values): - mock_values["network"] = {"dns_nameservers": ["nameserver1", "nameserver2"]} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["dns"] == ["nameserver1", "nameserver2"] - - -def test_add_duplicate_dns_nameservers(mock_values): - mock_values["network"] = {"dns_nameservers": ["nameserver1", "nameserver1"]} - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_add_duplicate_dns_searches(mock_values): - mock_values["network"] = {"dns_searches": ["search1", "search1"]} - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_add_duplicate_dns_opts(mock_values): - mock_values["network"] = {"dns_opts": ["attempts:3", "attempts:5"]} - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") diff --git a/library/2.0.33/tests/test_environment.py b/library/2.0.33/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/library/2.0.33/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/library/2.0.33/tests/test_formatter.py b/library/2.0.33/tests/test_formatter.py deleted file mode 100644 index 843cf65d2e..0000000000 --- a/library/2.0.33/tests/test_formatter.py +++ /dev/null @@ -1,13 +0,0 @@ -from formatter import escape_dollar - - -def test_escape_dollar(): - cases = [ - {"input": "test", "expected": "test"}, - {"input": "$test", "expected": "$$test"}, - {"input": "$$test", "expected": "$$$$test"}, - {"input": "$$$test", "expected": "$$$$$$test"}, - {"input": "$test$", "expected": "$$test$$"}, - ] - for case in cases: - assert escape_dollar(case["input"]) == case["expected"] diff --git a/library/2.0.33/tests/test_functions.py b/library/2.0.33/tests/test_functions.py deleted file mode 100644 index 0ea3b57d18..0000000000 --- a/library/2.0.33/tests/test_functions.py +++ /dev/null @@ -1,88 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - mock_values["ix_volumes"] = {"test": "/mnt/test123"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, - {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, - { - "func": "get_host_path", - "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], - "expected": "/mnt/test", - }, - { - "func": "get_host_path", - "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], - "expected": "/mnt/test123", - }, - {"func": "or_default", "values": [None, 1], "expected": 1}, - {"func": "or_default", "values": [1, None], "expected": 1}, - {"func": "or_default", "values": [False, 1], "expected": 1}, - {"func": "or_default", "values": [True, 1], "expected": True}, - {"func": "temp_config", "values": [""], "expect_raise": True}, - { - "func": "temp_config", - "values": ["test"], - "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, - }, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/library/2.0.33/tests/test_healthcheck.py b/library/2.0.33/tests/test_healthcheck.py deleted file mode 100644 index fbd488ece4..0000000000 --- a/library/2.0.33/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/library/2.0.33/tests/test_labels.py b/library/2.0.33/tests/test_labels.py deleted file mode 100644 index ffa21eceac..0000000000 --- a/library/2.0.33/tests/test_labels.py +++ /dev/null @@ -1,88 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_disallowed_label(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.labels.add_label("com.docker.compose.service", "test_service") - - -def test_add_duplicate_label(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.labels.add_label("my.custom.label", "test_value") - with pytest.raises(Exception): - c1.labels.add_label("my.custom.label", "test_value1") - - -def test_add_label_on_non_existing_container(mock_values): - mock_values["labels"] = [ - { - "key": "my.custom.label1", - "value": "test_value1", - "containers": ["test_container", "test_container2"], - }, - ] - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.render() - - -def test_add_label(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.labels.add_label("my.custom.label1", "test_value1") - c1.labels.add_label("my.custom.label2", "test_value2") - output = render.render() - assert output["services"]["test_container"]["labels"] == { - "my.custom.label1": "test_value1", - "my.custom.label2": "test_value2", - } - - -def test_auto_add_labels(mock_values): - mock_values["labels"] = [ - { - "key": "my.custom.label1", - "value": "test_value1", - "containers": ["test_container", "test_container2"], - }, - { - "key": "my.custom.label2", - "value": "test_value2", - "containers": ["test_container"], - }, - ] - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c2 = render.add_container("test_container2", "test_image") - c1.healthcheck.disable() - c2.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["labels"] == { - "my.custom.label1": "test_value1", - "my.custom.label2": "test_value2", - } - assert output["services"]["test_container2"]["labels"] == { - "my.custom.label1": "test_value1", - } diff --git a/library/2.0.33/tests/test_notes.py b/library/2.0.33/tests/test_notes.py deleted file mode 100644 index 3613445385..0000000000 --- a/library/2.0.33/tests/test_notes.py +++ /dev/null @@ -1,213 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "ix_context": { - "app_metadata": { - "name": "test_app", - } - }, - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_notes(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert ( - output["x-notes"] - == """# Welcome to TrueNAS SCALE - -Thank you for installing test_app! - -## Documentation - -Documentation for test_app can be found at https://www.truenas.com/docs. - -## Bug reports - -If you find a bug in this app, please file an issue at -https://ixsystems.atlassian.net or https://github.com/truenas/apps - -## Feature requests or improvements - -If you find a feature request for this app, please file an issue at -https://ixsystems.atlassian.net or https://github.com/truenas/apps -""" - ) - - -def test_notes_with_warnings(mock_values): - render = Render(mock_values) - render.notes.add_warning("this is not properly configured. fix it now!") - render.notes.add_warning("that is not properly configured. fix it later!") - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert ( - output["x-notes"] - == """# Welcome to TrueNAS SCALE - -Thank you for installing test_app! - -## Warnings - -- this is not properly configured. fix it now! -- that is not properly configured. fix it later! - -## Documentation - -Documentation for test_app can be found at https://www.truenas.com/docs. - -## Bug reports - -If you find a bug in this app, please file an issue at -https://ixsystems.atlassian.net or https://github.com/truenas/apps - -## Feature requests or improvements - -If you find a feature request for this app, please file an issue at -https://ixsystems.atlassian.net or https://github.com/truenas/apps -""" - ) - - -def test_notes_with_deprecations(mock_values): - render = Render(mock_values) - render.notes.add_deprecation("this is will be removed later. fix it now!") - render.notes.add_deprecation("that is will be removed later. fix it later!") - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert ( - output["x-notes"] - == """# Welcome to TrueNAS SCALE - -Thank you for installing test_app! - -## Deprecations - -- this is will be removed later. fix it now! -- that is will be removed later. fix it later! - -## Documentation - -Documentation for test_app can be found at https://www.truenas.com/docs. - -## Bug reports - -If you find a bug in this app, please file an issue at -https://ixsystems.atlassian.net or https://github.com/truenas/apps - -## Feature requests or improvements - -If you find a feature request for this app, please file an issue at -https://ixsystems.atlassian.net or https://github.com/truenas/apps -""" - ) - - -def test_notes_with_body(mock_values): - render = Render(mock_values) - render.notes.set_body( - """## Additional info - -Some info -some other info. -""" - ) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert ( - output["x-notes"] - == """# Welcome to TrueNAS SCALE - -Thank you for installing test_app! - -## Additional info - -Some info -some other info. - -## Documentation - -Documentation for test_app can be found at https://www.truenas.com/docs. - -## Bug reports - -If you find a bug in this app, please file an issue at -https://ixsystems.atlassian.net or https://github.com/truenas/apps - -## Feature requests or improvements - -If you find a feature request for this app, please file an issue at -https://ixsystems.atlassian.net or https://github.com/truenas/apps -""" - ) - - -def test_notes_all(mock_values): - render = Render(mock_values) - render.notes.add_warning("this is not properly configured. fix it now!") - render.notes.add_warning("that is not properly configured. fix it later!") - render.notes.add_deprecation("this is will be removed later. fix it now!") - render.notes.add_deprecation("that is will be removed later. fix it later!") - render.notes.set_body( - """## Additional info - -Some info -some other info. -""" - ) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert ( - output["x-notes"] - == """# Welcome to TrueNAS SCALE - -Thank you for installing test_app! - -## Warnings - -- this is not properly configured. fix it now! -- that is not properly configured. fix it later! - -## Deprecations - -- this is will be removed later. fix it now! -- that is will be removed later. fix it later! - -## Additional info - -Some info -some other info. - -## Documentation - -Documentation for test_app can be found at https://www.truenas.com/docs. - -## Bug reports - -If you find a bug in this app, please file an issue at -https://ixsystems.atlassian.net or https://github.com/truenas/apps - -## Feature requests or improvements - -If you find a feature request for this app, please file an issue at -https://ixsystems.atlassian.net or https://github.com/truenas/apps -""" - ) diff --git a/library/2.0.33/tests/test_portal.py b/library/2.0.33/tests/test_portal.py deleted file mode 100644 index aebd9425c9..0000000000 --- a/library/2.0.33/tests/test_portal.py +++ /dev/null @@ -1,75 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_no_portals(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["x-portals"] == [] - - -def test_add_portal(mock_values): - render = Render(mock_values) - render.portals.add_portal({"scheme": "http", "path": "/", "port": 8080}) - render.portals.add_portal({"name": "Other Portal", "scheme": "https", "path": "/", "port": 8443}) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["x-portals"] == [ - {"name": "Other Portal", "scheme": "https", "host": "0.0.0.0", "port": 8443, "path": "/"}, - {"name": "Web UI", "scheme": "http", "host": "0.0.0.0", "port": 8080, "path": "/"}, - ] - - -def test_add_duplicate_portal(mock_values): - render = Render(mock_values) - render.portals.add_portal({"scheme": "http", "path": "/", "port": 8080}) - with pytest.raises(Exception): - render.portals.add_portal({"scheme": "http", "path": "/", "port": 8080}) - - -def test_add_duplicate_portal_with_explicit_name(mock_values): - render = Render(mock_values) - render.portals.add_portal({"name": "Some Portal", "scheme": "http", "path": "/", "port": 8080}) - with pytest.raises(Exception): - render.portals.add_portal({"name": "Some Portal", "scheme": "http", "path": "/", "port": 8080}) - - -def test_add_portal_with_invalid_scheme(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.portals.add_portal({"scheme": "invalid_scheme", "path": "/", "port": 8080}) - - -def test_add_portal_with_invalid_path(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.portals.add_portal({"scheme": "http", "path": "invalid_path", "port": 8080}) - - -def test_add_portal_with_invalid_path_double_slash(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.portals.add_portal({"scheme": "http", "path": "/some//path", "port": 8080}) - - -def test_add_portal_with_invalid_port(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.portals.add_portal({"scheme": "http", "path": "/", "port": -1}) diff --git a/library/2.0.33/tests/test_ports.py b/library/2.0.33/tests/test_ports.py deleted file mode 100644 index a4c923ca1d..0000000000 --- a/library/2.0.33/tests/test_ports.py +++ /dev/null @@ -1,110 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_ports(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.ports.add_port(8081, 8080) - c1.ports.add_port(8082, 8080, {"protocol": "udp"}) - output = render.render() - assert output["services"]["test_container"]["ports"] == [ - {"published": 8081, "target": 8080, "protocol": "tcp", "mode": "ingress", "host_ip": "0.0.0.0"}, - {"published": 8082, "target": 8080, "protocol": "udp", "mode": "ingress", "host_ip": "0.0.0.0"}, - ] - - -def test_add_duplicate_ports(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.ports.add_port(8081, 8080) - c1.ports.add_port(8081, 8080, {"protocol": "udp"}) # This should not raise - with pytest.raises(Exception): - c1.ports.add_port(8081, 8080) - - -def test_add_duplicate_ports_with_different_host_ip(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.ports.add_port(8081, 8080, {"host_ip": "192.168.1.10"}) - c1.ports.add_port(8081, 8080, {"host_ip": "192.168.1.11"}) - output = render.render() - assert output["services"]["test_container"]["ports"] == [ - {"published": 8081, "target": 8080, "protocol": "tcp", "mode": "ingress", "host_ip": "192.168.1.10"}, - {"published": 8081, "target": 8080, "protocol": "tcp", "mode": "ingress", "host_ip": "192.168.1.11"}, - ] - - -def test_add_duplicate_ports_to_specific_host_ip_binds_to_0_0_0_0(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.ports.add_port(8081, 8080, {"host_ip": "192.168.1.10"}) - with pytest.raises(Exception): - c1.ports.add_port(8081, 8080, {"host_ip": "0.0.0.0"}) - - -def test_add_duplicate_ports_to_0_0_0_0_binds_to_specific_host_ip(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.ports.add_port(8081, 8080, {"host_ip": "0.0.0.0"}) - with pytest.raises(Exception): - c1.ports.add_port(8081, 8080, {"host_ip": "192.168.1.10"}) - - -def test_add_ports_with_invalid_protocol(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.ports.add_port(8081, 8080, {"protocol": "invalid_protocol"}) - - -def test_add_ports_with_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.ports.add_port(8081, 8080, {"mode": "invalid_mode"}) - - -def test_add_ports_with_invalid_ip(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.ports.add_port(8081, 8080, {"host_ip": "invalid_ip"}) - - -def test_add_ports_with_invalid_host_port(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.ports.add_port(-1, 8080) - - -def test_add_ports_with_invalid_container_port(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.ports.add_port(8081, -1) diff --git a/library/2.0.33/tests/test_render.py b/library/2.0.33/tests/test_render.py deleted file mode 100644 index 60dc00679e..0000000000 --- a/library/2.0.33/tests/test_render.py +++ /dev/null @@ -1,37 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_values_cannot_be_modified(mock_values): - render = Render(mock_values) - render.values["test"] = "test" - with pytest.raises(Exception): - render.render() - - -def test_duplicate_containers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_no_containers(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.render() diff --git a/library/2.0.33/tests/test_resources.py b/library/2.0.33/tests/test_resources.py deleted file mode 100644 index cd83d164e5..0000000000 --- a/library/2.0.33/tests/test_resources.py +++ /dev/null @@ -1,140 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_automatically_add_cpu(mock_values): - mock_values["resources"] = {"limits": {"cpus": 1.0}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["deploy"]["resources"]["limits"]["cpus"] == "1.0" - - -def test_invalid_cpu(mock_values): - mock_values["resources"] = {"limits": {"cpus": "invalid"}} - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_automatically_add_memory(mock_values): - mock_values["resources"] = {"limits": {"memory": 1024}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["deploy"]["resources"]["limits"]["memory"] == "1024M" - - -def test_invalid_memory(mock_values): - mock_values["resources"] = {"limits": {"memory": "invalid"}} - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_automatically_add_gpus(mock_values): - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - devices = output["services"]["test_container"]["deploy"]["resources"]["reservations"]["devices"] - assert len(devices) == 1 - assert devices[0] == { - "capabilities": ["gpu"], - "driver": "nvidia", - "device_ids": ["uuid_0", "uuid_1"], - } - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_gpu_without_uuid(mock_values): - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_remove_cpus_and_memory_with_gpus(mock_values): - mock_values["resources"] = {"gpus": {"nvidia_gpu_selection": {"pci_slot_0": {"uuid": "uuid_1", "use_gpu": True}}}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.deploy.resources.remove_cpus_and_memory() - output = render.render() - assert "limits" not in output["services"]["test_container"]["deploy"]["resources"] - devices = output["services"]["test_container"]["deploy"]["resources"]["reservations"]["devices"] - assert len(devices) == 1 - assert devices[0] == { - "capabilities": ["gpu"], - "driver": "nvidia", - "device_ids": ["uuid_1"], - } - - -def test_remove_cpus_and_memory(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.deploy.resources.remove_cpus_and_memory() - output = render.render() - assert "deploy" not in output["services"]["test_container"] - - -def test_remove_devices(mock_values): - mock_values["resources"] = {"gpus": {"nvidia_gpu_selection": {"pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}}}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.deploy.resources.remove_devices() - output = render.render() - assert "reservations" not in output["services"]["test_container"]["deploy"]["resources"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_set_profile(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.deploy.resources.set_profile("low") - output = render.render() - assert output["services"]["test_container"]["deploy"]["resources"]["limits"]["cpus"] == "1" - assert output["services"]["test_container"]["deploy"]["resources"]["limits"]["memory"] == "512M" - - -def test_set_profile_invalid_profile(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.deploy.resources.set_profile("invalid_profile") diff --git a/library/2.0.33/tests/test_restart.py b/library/2.0.33/tests/test_restart.py deleted file mode 100644 index 06b2975590..0000000000 --- a/library/2.0.33/tests/test_restart.py +++ /dev/null @@ -1,57 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_invalid_restart_policy(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.restart.set_policy("invalid_policy") - - -def test_valid_restart_policy(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.restart.set_policy("on-failure") - output = render.render() - assert output["services"]["test_container"]["restart"] == "on-failure" - - -def test_valid_restart_policy_with_maximum_retry_count(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.restart.set_policy("on-failure", 10) - output = render.render() - assert output["services"]["test_container"]["restart"] == "on-failure:10" - - -def test_invalid_restart_policy_with_maximum_retry_count(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.restart.set_policy("on-failure", maximum_retry_count=-1) - - -def test_invalid_restart_policy_with_maximum_retry_count_and_policy(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.restart.set_policy("always", maximum_retry_count=10) diff --git a/library/2.0.33/tests/test_sysctls.py b/library/2.0.33/tests/test_sysctls.py deleted file mode 100644 index c9414044ea..0000000000 --- a/library/2.0.33/tests/test_sysctls.py +++ /dev/null @@ -1,62 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_sysctl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.sysctls.add("net.ipv4.ip_forward", 1) - c1.sysctls.add("fs.mqueue.msg_max", 100) - output = render.render() - assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} - - -def test_add_net_sysctl_with_host_network(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - c1.sysctls.add("net.ipv4.ip_forward", 1) - with pytest.raises(Exception): - render.render() - - -def test_add_duplicate_sysctl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.sysctls.add("net.ipv4.ip_forward", 1) - with pytest.raises(Exception): - c1.sysctls.add("net.ipv4.ip_forward", 0) - - -def test_add_empty_sysctl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.sysctls.add("", 1) - - -def test_add_sysctl_with_invalid_key(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.sysctls.add("invalid.sysctl", 1) - with pytest.raises(Exception): - render.render() diff --git a/library/2.0.33/tests/test_volumes.py b/library/2.0.33/tests/test_volumes.py deleted file mode 100644 index aef0d39481..0000000000 --- a/library/2.0.33/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage._add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/library/2.0.33/validations.py b/library/2.0.33/validations.py deleted file mode 100644 index b0dea174b7..0000000000 --- a/library/2.0.33/validations.py +++ /dev/null @@ -1,227 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_sysctl_or_raise(sysctl: str, host_network: bool): - if not sysctl: - raise RenderError("Sysctl cannot be empty") - if host_network and sysctl.startswith("net."): - raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") - - valid_sysctls = [ - "kernel.msgmax", - "kernel.msgmnb", - "kernel.msgmni", - "kernel.sem", - "kernel.shmall", - "kernel.shmmax", - "kernel.shmmni", - "kernel.shm_rmid_forced", - ] - # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls - if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: - raise RenderError( - f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" - ) - return sysctl - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/library/2.0.33/volume_mount.py b/library/2.0.33/volume_mount.py deleted file mode 100644 index aadd077750..0000000000 --- a/library/2.0.33/volume_mount.py +++ /dev/null @@ -1,92 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .formatter import merge_dicts_no_overwrite - from .volume_mount_types import BindMountType, VolumeMountType, TmpfsMountType - from .volume_sources import HostPathSource, IxVolumeSource, CifsSource, NfsSource, VolumeSource -except ImportError: - from error import RenderError - from formatter import merge_dicts_no_overwrite - from volume_mount_types import BindMountType, VolumeMountType, TmpfsMountType - from volume_sources import HostPathSource, IxVolumeSource, CifsSource, NfsSource, VolumeSource - - -class VolumeMount: - def __init__(self, render_instance: "Render", mount_path: str, config: "IxStorage"): - self._render_instance = render_instance - self.mount_path: str = mount_path - - storage_type: str = config.get("type", "") - if not storage_type: - raise RenderError("Expected [type] to be set for volume mounts.") - - match storage_type: - case "host_path": - spec_type = "bind" - mount_config = config.get("host_path_config") - if mount_config is None: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - mount_type_specific_definition = BindMountType(self._render_instance, mount_config).render() - source = HostPathSource(self._render_instance, mount_config).get() - case "ix_volume": - spec_type = "bind" - mount_config = config.get("ix_volume_config") - if mount_config is None: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - mount_type_specific_definition = BindMountType(self._render_instance, mount_config).render() - source = IxVolumeSource(self._render_instance, mount_config).get() - case "tmpfs": - spec_type = "tmpfs" - mount_config = config.get("tmpfs_config", {}) - mount_type_specific_definition = TmpfsMountType(self._render_instance, mount_config).render() - source = None - case "nfs": - spec_type = "volume" - mount_config = config.get("nfs_config") - if mount_config is None: - raise RenderError("Expected [nfs_config] to be set for [nfs] type.") - mount_type_specific_definition = VolumeMountType(self._render_instance, mount_config).render() - source = NfsSource(self._render_instance, mount_config).get() - case "cifs": - spec_type = "volume" - mount_config = config.get("cifs_config") - if mount_config is None: - raise RenderError("Expected [cifs_config] to be set for [cifs] type.") - mount_type_specific_definition = VolumeMountType(self._render_instance, mount_config).render() - source = CifsSource(self._render_instance, mount_config).get() - case "volume": - spec_type = "volume" - mount_config = config.get("volume_config") - if mount_config is None: - raise RenderError("Expected [volume_config] to be set for [volume] type.") - mount_type_specific_definition = VolumeMountType(self._render_instance, mount_config).render() - source = VolumeSource(self._render_instance, mount_config).get() - case "temporary": - spec_type = "volume" - mount_config = config.get("volume_config") - if mount_config is None: - raise RenderError("Expected [volume_config] to be set for [temporary] type.") - mount_type_specific_definition = VolumeMountType(self._render_instance, mount_config).render() - source = VolumeSource(self._render_instance, mount_config).get() - case "anonymous": - spec_type = "volume" - mount_config = config.get("volume_config") or {} - mount_type_specific_definition = VolumeMountType(self._render_instance, mount_config).render() - source = None - case _: - raise RenderError(f"Storage type [{storage_type}] is not supported for volume mounts.") - - common_spec = {"type": spec_type, "target": self.mount_path, "read_only": config.get("read_only", False)} - if source is not None: - common_spec["source"] = source - self._render_instance.volumes.add_volume(source, storage_type, mount_config) # type: ignore - - self.volume_mount_spec = merge_dicts_no_overwrite(common_spec, mount_type_specific_definition) - - def render(self) -> dict: - return self.volume_mount_spec diff --git a/library/2.0.33/volume_mount_types.py b/library/2.0.33/volume_mount_types.py deleted file mode 100644 index 00a0ec3a18..0000000000 --- a/library/2.0.33/volume_mount_types.py +++ /dev/null @@ -1,72 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorageTmpfsConfig, IxStorageVolumeConfig, IxStorageBindLikeConfigs - - -try: - from .error import RenderError - from .validations import valid_host_path_propagation, valid_octal_mode_or_raise -except ImportError: - from error import RenderError - from validations import valid_host_path_propagation, valid_octal_mode_or_raise - - -class TmpfsMountType: - def __init__(self, render_instance: "Render", config: "IxStorageTmpfsConfig"): - self._render_instance = render_instance - self.spec = {"tmpfs": {}} - size = config.get("size", None) - mode = config.get("mode", None) - - if size is not None: - if not isinstance(size, int): - raise RenderError(f"Expected [size] to be an integer for [tmpfs] type, got [{size}]") - if not size > 0: - raise RenderError(f"Expected [size] to be greater than 0 for [tmpfs] type, got [{size}]") - # Convert Mebibytes to Bytes - self.spec["tmpfs"]["size"] = size * 1024 * 1024 - - if mode is not None: - mode = valid_octal_mode_or_raise(mode) - self.spec["tmpfs"]["mode"] = int(mode, 8) - - if not self.spec["tmpfs"]: - self.spec.pop("tmpfs") - - def render(self) -> dict: - """Render the tmpfs mount specification.""" - return self.spec - - -class BindMountType: - def __init__(self, render_instance: "Render", config: "IxStorageBindLikeConfigs"): - self._render_instance = render_instance - self.spec: dict = {} - - propagation = valid_host_path_propagation(config.get("propagation", "rprivate")) - create_host_path = config.get("create_host_path", False) - - self.spec: dict = { - "bind": { - "create_host_path": create_host_path, - "propagation": propagation, - } - } - - def render(self) -> dict: - """Render the bind mount specification.""" - return self.spec - - -class VolumeMountType: - def __init__(self, render_instance: "Render", config: "IxStorageVolumeConfig"): - self._render_instance = render_instance - self.spec: dict = {} - - self.spec: dict = {"volume": {"nocopy": config.get("nocopy", False)}} - - def render(self) -> dict: - """Render the volume mount specification.""" - return self.spec diff --git a/library/2.0.33/volume_sources.py b/library/2.0.33/volume_sources.py deleted file mode 100644 index c33fe55ea1..0000000000 --- a/library/2.0.33/volume_sources.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorageHostPathConfig, IxStorageIxVolumeConfig, IxStorageVolumeConfig - -try: - from .error import RenderError - from .formatter import get_hashed_name_for_volume - from .validations import valid_fs_path_or_raise -except ImportError: - from error import RenderError - from formatter import get_hashed_name_for_volume - from validations import valid_fs_path_or_raise - - -class HostPathSource: - def __init__(self, render_instance: "Render", config: "IxStorageHostPathConfig"): - self._render_instance = render_instance - self.source: str = "" - - if not config: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - - path = "" - if config.get("acl_enable", False): - acl_path = config.get("acl", {}).get("path") - if not acl_path: - raise RenderError("Expected [host_path_config.acl.path] to be set for [host_path] type.") - path = valid_fs_path_or_raise(acl_path) - else: - path = valid_fs_path_or_raise(config.get("path", "")) - - self.source = path.rstrip("/") - - def get(self): - return self.source - - -class IxVolumeSource: - def __init__(self, render_instance: "Render", config: "IxStorageIxVolumeConfig"): - self._render_instance = render_instance - self.source: str = "" - - if not config: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - dataset_name = config.get("dataset_name") - if not dataset_name: - raise RenderError("Expected [ix_volume_config.dataset_name] to be set for [ix_volume] type.") - - ix_volumes = self._render_instance.values.get("ix_volumes", {}) - if dataset_name not in ix_volumes: - available = ", ".join(ix_volumes.keys()) - raise RenderError( - f"Expected the key [{dataset_name}] to be set in [ix_volumes] for [ix_volume] type. " - f"Available keys: [{available}]." - ) - - self.source = valid_fs_path_or_raise(ix_volumes[dataset_name].rstrip("/")) - - def get(self): - return self.source - - -class CifsSource: - def __init__(self, render_instance: "Render", config: dict): - self._render_instance = render_instance - self.source: str = "" - - if not config: - raise RenderError("Expected [cifs_config] to be set for [cifs] type.") - self.source = get_hashed_name_for_volume("cifs", config) - - def get(self): - return self.source - - -class NfsSource: - def __init__(self, render_instance: "Render", config: dict): - self._render_instance = render_instance - self.source: str = "" - - if not config: - raise RenderError("Expected [nfs_config] to be set for [nfs] type.") - self.source = get_hashed_name_for_volume("nfs", config) - - def get(self): - return self.source - - -class VolumeSource: - def __init__(self, render_instance: "Render", config: "IxStorageVolumeConfig"): - self._render_instance = render_instance - self.source: str = "" - - if not config: - raise RenderError("Expected [volume_config] to be set for [volume] type.") - - volume_name: str = config.get("volume_name", "") - if not volume_name: - raise RenderError("Expected [volume_config.volume_name] to be set for [volume] type.") - - self.source = volume_name - - def get(self): - return self.source diff --git a/library/2.0.33/volume_types.py b/library/2.0.33/volume_types.py deleted file mode 100644 index 4ccea08f83..0000000000 --- a/library/2.0.33/volume_types.py +++ /dev/null @@ -1,133 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorageNfsConfig, IxStorageCifsConfig, IxStorageVolumeConfig - - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_fs_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_fs_path_or_raise - - -class NfsVolume: - def __init__(self, render_instance: "Render", config: "IxStorageNfsConfig"): - self._render_instance = render_instance - - if not config: - raise RenderError("Expected [nfs_config] to be set for [nfs] type") - - required_keys = ["server", "path"] - for key in required_keys: - if not config.get(key): - raise RenderError(f"Expected [{key}] to be set for [nfs] type") - - opts = [f"addr={config['server']}"] - cfg_options = config.get("options") - if cfg_options: - if not isinstance(cfg_options, list): - raise RenderError("Expected [nfs_config.options] to be a list for [nfs] type") - - tracked_keys: set[str] = set() - disallowed_opts = ["addr"] - for opt in cfg_options: - if not isinstance(opt, str): - raise RenderError("Options for [nfs] type must be a list of strings.") - - key = opt.split("=")[0] - if key in tracked_keys: - raise RenderError(f"Option [{key}] already added for [nfs] type.") - if key in disallowed_opts: - raise RenderError(f"Option [{key}] is not allowed for [nfs] type.") - opts.append(opt) - tracked_keys.add(key) - - opts.sort() - - path = valid_fs_path_or_raise(config["path"].rstrip("/")) - self.volume_spec = { - "driver_opts": { - "type": "nfs", - "device": f":{path}", - "o": f"{','.join([escape_dollar(opt) for opt in opts])}", - }, - } - - def get(self): - return self.volume_spec - - -class CifsVolume: - def __init__(self, render_instance: "Render", config: "IxStorageCifsConfig"): - self._render_instance = render_instance - self.volume_spec: dict = {} - - if not config: - raise RenderError("Expected [cifs_config] to be set for [cifs] type") - - required_keys = ["server", "path", "username", "password"] - for key in required_keys: - if not config.get(key): - raise RenderError(f"Expected [{key}] to be set for [cifs] type") - - opts = [ - "noperm", - f"user={config['username']}", - f"password={config['password']}", - ] - - domain = config.get("domain") - if domain: - opts.append(f"domain={domain}") - - cfg_options = config.get("options") - if cfg_options: - if not isinstance(cfg_options, list): - raise RenderError("Expected [cifs_config.options] to be a list for [cifs] type") - - tracked_keys: set[str] = set() - disallowed_opts = ["user", "password", "domain", "noperm"] - for opt in cfg_options: - if not isinstance(opt, str): - raise RenderError("Options for [cifs] type must be a list of strings.") - - key = opt.split("=")[0] - if key in tracked_keys: - raise RenderError(f"Option [{key}] already added for [cifs] type.") - if key in disallowed_opts: - raise RenderError(f"Option [{key}] is not allowed for [cifs] type.") - for disallowed in disallowed_opts: - if key == disallowed: - raise RenderError(f"Option [{key}] is not allowed for [cifs] type.") - opts.append(opt) - tracked_keys.add(key) - opts.sort() - - server = config["server"].lstrip("/") - path = config["path"].strip("/") - path = valid_fs_path_or_raise("/" + path).lstrip("/") - - self.volume_spec = { - "driver_opts": { - "type": "cifs", - "device": f"//{server}/{path}", - "o": f"{','.join([escape_dollar(opt) for opt in opts])}", - }, - } - - def get(self): - return self.volume_spec - - -class DockerVolume: - def __init__(self, render_instance: "Render", config: "IxStorageVolumeConfig"): - self._render_instance = render_instance - self.volume_spec: dict = {} - - def get(self): - return self.volume_spec diff --git a/library/2.0.33/volumes.py b/library/2.0.33/volumes.py deleted file mode 100644 index e6925a402f..0000000000 --- a/library/2.0.33/volumes.py +++ /dev/null @@ -1,66 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - - -try: - from .error import RenderError - from .storage import IxStorageVolumeLikeConfigs - from .volume_types import NfsVolume, CifsVolume, DockerVolume -except ImportError: - from error import RenderError - from storage import IxStorageVolumeLikeConfigs - from volume_types import NfsVolume, CifsVolume, DockerVolume - - -class Volumes: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volumes: dict[str, Volume] = {} - - def add_volume( - self, - source: str, - storage_type: str, - config: "IxStorageVolumeLikeConfigs", - ): - # This method can be called many times from the volume mounts - # Only add the volume if it is not already added, but dont raise an error - if source == "": - raise RenderError(f"Volume source [{source}] cannot be empty") - - if source in self._volumes: - return - - self._volumes[source] = Volume(self._render_instance, storage_type, config) - - def has_volumes(self) -> bool: - return bool(self._volumes) - - def render(self): - return {name: v.render() for name, v in sorted(self._volumes.items()) if v.render() is not None} - - -class Volume: - def __init__( - self, - render_instance: "Render", - storage_type: str, - config: "IxStorageVolumeLikeConfigs", - ): - self._render_instance = render_instance - self.volume_spec: dict | None = {} - - match storage_type: - case "nfs": - self.volume_spec = NfsVolume(self._render_instance, config).get() # type: ignore - case "cifs": - self.volume_spec = CifsVolume(self._render_instance, config).get() # type: ignore - case "volume" | "temporary": - self.volume_spec = DockerVolume(self._render_instance, config).get() # type: ignore - case _: - self.volume_spec = None - - def render(self): - return self.volume_spec