From 3f86b6b2e94446dafda47e13a9ad1ebcd2c590a2 Mon Sep 17 00:00:00 2001 From: Felix Rubio Date: Wed, 27 Nov 2024 21:52:45 +0100 Subject: [PATCH 1/8] First attempt at providing support for conditional dependencies Signed-off-by: Felix Rubio --- podman_compose.py | 110 +++++++++++++----- .../deps/docker-compose-conditional.yaml | 22 ++++ tests/integration/test_podman_compose_deps.py | 32 ++++- 3 files changed, 130 insertions(+), 34 deletions(-) create mode 100644 tests/integration/deps/docker-compose-conditional.yaml diff --git a/podman_compose.py b/podman_compose.py index db6dc58e..de914fd6 100755 --- a/podman_compose.py +++ b/podman_compose.py @@ -25,6 +25,7 @@ import subprocess import sys from asyncio import Task +from enum import Enum try: from shlex import quote as cmd_quote @@ -1048,8 +1049,8 @@ async def container_to_args(compose, cnt, detached=True): if pod: podman_args.append(f"--pod={pod}") deps = [] - for dep_srv in cnt.get("_deps", None) or []: - deps.extend(compose.container_names_by_service.get(dep_srv, None) or []) + for dep_srv in cnt.get("_deps", []): + deps.extend(compose.container_names_by_service.get(dep_srv.name, [])) if deps: deps_csv = ",".join(deps) podman_args.append(f"--requires={deps_csv}") @@ -1273,33 +1274,63 @@ async def container_to_args(compose, cnt, detached=True): return podman_args -def rec_deps(services, service_name, start_point=None): - """ - return all dependencies of service_name recursively - """ - if not start_point: - start_point = service_name - deps = services[service_name]["_deps"] - for dep_name in deps.copy(): - # avoid A depens on A - if dep_name == service_name: - continue - dep_srv = services.get(dep_name, None) - if not dep_srv: - continue - # NOTE: avoid creating loops, A->B->A - if start_point and start_point in dep_srv["_deps"]: - continue - new_deps = rec_deps(services, dep_name, start_point) - deps.update(new_deps) - return deps +class ServiceDependencyCondition(Enum): + NONE = "" + HEALTHY = "service_healthy" + STARTED = "service_started" + COMPLETED = "service_completed_successfully" + + +class ServiceDependency: + def __init__(self, name, condition=None): + self._name = name + self._condition = ServiceDependencyCondition(condition or "") + + @property + def name(self): + return self._name + + @property + def condition(self): + return self._condition + + def __hash__(self): + # Compute hash based on the frozenset of items to ensure order does not matter + return hash(('name', self._name)+('condition', self._condition)) + + def __eq__(self, other): + # Compare equality based on dictionary content + if isinstance(other, ServiceDependency): + return self._name == other.name and self._condition == other.condition + return False def flat_deps(services, with_extends=False): """ create dependencies "_deps" or update it recursively for all services """ + def rec_deps(services, service_name, start_point=None): + """ + return all dependencies of service_name recursively + """ + start_point = start_point or service_name + deps = services[service_name]["_deps"] + for dep_name in deps.copy(): + # avoid A depens on A + if dep_name.name == service_name: + continue + dep_srv = services.get(dep_name.name, None) + if not dep_srv: + continue + # NOTE: avoid creating loops, A->B->A + if start_point in any(x.name for x in dep_srv["_deps"]): + continue + new_deps = rec_deps(services, dep_name, start_point) + deps.update(new_deps) + return deps + for name, srv in services.items(): + # parse dependencies for each service deps = set() srv["_deps"] = deps if with_extends: @@ -1308,14 +1339,16 @@ def flat_deps(services, with_extends=False): if ext != name: deps.add(ext) continue - deps_ls = srv.get("depends_on", None) or [] - if isinstance(deps_ls, str): - deps_ls = [deps_ls] + deps_ls = srv.get("depends_on", []) + if isinstance(deps_ls, list): + deps_ls = [ServiceDependency(t) for t in deps_ls] elif isinstance(deps_ls, dict): - deps_ls = list(deps_ls.keys()) - deps.update(deps_ls) + deps_ls = [ServiceDependency(k, v.get("condition")) for k, v in deps.items()] + else: + raise RuntimeError("depends_on should be a list of strings or a dict") + # parse link to get service name and remove alias - links_ls = srv.get("links", None) or [] + links_ls = srv.get("links", []) if not is_list(links_ls): links_ls = [links_ls] deps.update([(c.split(":")[0] if ":" in c else c) for c in links_ls]) @@ -1325,6 +1358,8 @@ def flat_deps(services, with_extends=False): if "_aliases" not in services[dep_name]: services[dep_name]["_aliases"] = set() services[dep_name]["_aliases"].add(dep_alias) + + # expand the dependencies on each service for name, srv in services.items(): rec_deps(services, name) @@ -2022,7 +2057,7 @@ def _parse_compose_file(self): container_by_name = {c["name"]: c for c in given_containers} # log("deps:", [(c["name"], c["_deps"]) for c in given_containers]) given_containers = list(container_by_name.values()) - given_containers.sort(key=lambda c: len(c.get("_deps", None) or [])) + given_containers.sort(key=lambda c: len(c.get("_deps", []))) # log("sorted:", [c["name"] for c in given_containers]) self.x_podman = compose.get("x-podman", {}) @@ -2496,7 +2531,7 @@ def get_excluded(compose, args): if args.services: excluded = set(compose.services) for service in args.services: - excluded -= compose.services[service]["_deps"] + excluded -= set(x.name for x in compose.services[service]["_deps"]) excluded.discard(service) log.debug("** excluding: %s", excluded) return excluded @@ -2732,7 +2767,7 @@ async def compose_run(compose, args): **dict( args.__dict__, detach=True, - services=deps, + services=[x.name for x in deps], # defaults no_build=False, build=None, @@ -2751,6 +2786,19 @@ async def compose_run(compose, args): ) await compose.commands["build"](compose, build_args) + # Separate the dependencies into different lists based on their condition + deps_healthy = [d.name for d in deps if d.condition == ServiceDependencyCondition.HEALTHY] + deps_started = [d.name for d in deps if d.condition == ServiceDependencyCondition.STARTED] + deps_completed = [d.name for d in deps if d.condition == ServiceDependencyCondition.COMPLETED] + + # execute podman wait on the dependencies + if deps_started: + await compose.podman.run([], "wait", ["--condition=running"] + deps_started) + if deps_healthy: + await compose.podman.run([], "wait", ["--condition=healthy"] + deps_healthy) + if deps_completed: + await compose.podman.run([], "wait", deps_completed) + compose_run_update_container_from_args(compose, cnt, args) # run podman podman_args = await container_to_args(compose, cnt, args.detach) diff --git a/tests/integration/deps/docker-compose-conditional.yaml b/tests/integration/deps/docker-compose-conditional.yaml new file mode 100644 index 00000000..bd110b71 --- /dev/null +++ b/tests/integration/deps/docker-compose-conditional.yaml @@ -0,0 +1,22 @@ +version: "3.7" +services: + web: + image: nopush/podman-compose-test + command: ["dumb-init", "/bin/busybox", "httpd", "-f", "-h", "/etc/", "-p", "8000"] + tmpfs: + - /run + - /tmp + healthcheck: + test: ["CMD", "wget", "-qO-", "http://localhost:8000/hosts"] + interval: 30s # Time between health checks + timeout: 5s # Time to wait for a response + retries: 3 # Number of consecutive failures before marking as unhealthy + sleep: + image: nopush/podman-compose-test + command: ["dumb-init", "/bin/busybox", "sh", "-c", "sleep 3600"] + depends_on: + web: + condition: service_healthy + tmpfs: + - /run + - /tmp diff --git a/tests/integration/test_podman_compose_deps.py b/tests/integration/test_podman_compose_deps.py index 1c468121..3fdd7754 100644 --- a/tests/integration/test_podman_compose_deps.py +++ b/tests/integration/test_podman_compose_deps.py @@ -7,11 +7,11 @@ from tests.integration.test_utils import RunSubprocessMixin -def compose_yaml_path(): - return os.path.join(os.path.join(test_path(), "deps"), "docker-compose.yaml") +def compose_yaml_path(suffix=""): + return os.path.join(os.path.join(test_path(), "deps"), f"docker-compose{suffix}.yaml") -class TestComposeDeps(unittest.TestCase, RunSubprocessMixin): +class TestComposeBaseDeps(unittest.TestCase, RunSubprocessMixin): def test_deps(self): try: output, error = self.run_subprocess_assert_returncode([ @@ -34,3 +34,29 @@ def test_deps(self): compose_yaml_path(), "down", ]) + + +class TestComposeConditionalDeps(unittest.TestCase, RunSubprocessMixin): + def test_deps(self): + suffix = "-conditional" + try: + output, error = self.run_subprocess_assert_returncode([ + podman_compose_path(), + "-f", + compose_yaml_path(suffix), + "run", + "--rm", + "sleep", + "/bin/sh", + "-c", + "wget -O - http://web:8000/hosts", + ]) + self.assertIn(b"HTTP request sent, awaiting response... 200 OK", output) + self.assertIn(b"deps_web_1", output) + finally: + self.run_subprocess_assert_returncode([ + podman_compose_path(), + "-f", + compose_yaml_path(suffix), + "down", + ]) From ec219cab951a19e074fdf23092c64f44c68dfd2e Mon Sep 17 00:00:00 2001 From: Felix Rubio Date: Wed, 27 Nov 2024 21:58:34 +0100 Subject: [PATCH 2/8] included newsfragment Signed-off-by: Felix Rubio --- newsfragments/conditional-dependencies.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/conditional-dependencies.feature diff --git a/newsfragments/conditional-dependencies.feature b/newsfragments/conditional-dependencies.feature new file mode 100644 index 00000000..78bf8582 --- /dev/null +++ b/newsfragments/conditional-dependencies.feature @@ -0,0 +1 @@ +Added support for honoring the condition in the depends_on section of the service, if stated. \ No newline at end of file From 34df9e186c3480a70deb093ff3fd219138b75d96 Mon Sep 17 00:00:00 2001 From: Felix Rubio Date: Fri, 29 Nov 2024 09:48:01 +0100 Subject: [PATCH 3/8] Bringing back the case of a single string stated in depends_on. Signed-off-by: Felix Rubio --- podman_compose.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/podman_compose.py b/podman_compose.py index de914fd6..b0004be9 100755 --- a/podman_compose.py +++ b/podman_compose.py @@ -1340,7 +1340,9 @@ def rec_deps(services, service_name, start_point=None): deps.add(ext) continue deps_ls = srv.get("depends_on", []) - if isinstance(deps_ls, list): + if isinstance(deps_ls, str): + deps_ls = [ServiceDependency(dep_ls)] + elif isinstance(deps_ls, list): deps_ls = [ServiceDependency(t) for t in deps_ls] elif isinstance(deps_ls, dict): deps_ls = [ServiceDependency(k, v.get("condition")) for k, v in deps.items()] From 19b5896c8f405526d5b72fcad4f55026a5211d1f Mon Sep 17 00:00:00 2001 From: Felix Rubio Date: Fri, 29 Nov 2024 10:06:07 +0100 Subject: [PATCH 4/8] Adding management of extending files Signed-off-by: Felix Rubio --- podman_compose.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/podman_compose.py b/podman_compose.py index b0004be9..14884f3b 100755 --- a/podman_compose.py +++ b/podman_compose.py @@ -1337,7 +1337,7 @@ def rec_deps(services, service_name, start_point=None): ext = srv.get("extends", {}).get("service", None) if ext: if ext != name: - deps.add(ext) + deps.add(ServiceDependency(ext)) continue deps_ls = srv.get("depends_on", []) if isinstance(deps_ls, str): From 7938f9a0ccf4170fcf99ead1d9f9a70e85c94413 Mon Sep 17 00:00:00 2001 From: Felix Rubio Date: Fri, 29 Nov 2024 20:54:08 +0100 Subject: [PATCH 5/8] fixed wrong variable used in dependency parsing Signed-off-by: Felix Rubio --- podman_compose.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/podman_compose.py b/podman_compose.py index 14884f3b..cce32d76 100755 --- a/podman_compose.py +++ b/podman_compose.py @@ -1345,9 +1345,9 @@ def rec_deps(services, service_name, start_point=None): elif isinstance(deps_ls, list): deps_ls = [ServiceDependency(t) for t in deps_ls] elif isinstance(deps_ls, dict): - deps_ls = [ServiceDependency(k, v.get("condition")) for k, v in deps.items()] + deps_ls = [ServiceDependency(k, v.get("condition")) for k, v in deps_ls.items()] else: - raise RuntimeError("depends_on should be a list of strings or a dict") + raise RuntimeError("depends_on should be a string, a list of strings or a dict") # parse link to get service name and remove alias links_ls = srv.get("links", []) From f660a0ccf48688c45bb447e270118c935de91303 Mon Sep 17 00:00:00 2001 From: Felix Rubio Date: Fri, 29 Nov 2024 21:02:06 +0100 Subject: [PATCH 6/8] fixed the search for object with given start point name within deps set Signed-off-by: Felix Rubio --- podman_compose.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/podman_compose.py b/podman_compose.py index cce32d76..d03b829d 100755 --- a/podman_compose.py +++ b/podman_compose.py @@ -1323,7 +1323,7 @@ def rec_deps(services, service_name, start_point=None): if not dep_srv: continue # NOTE: avoid creating loops, A->B->A - if start_point in any(x.name for x in dep_srv["_deps"]): + if any(start_point == x.name for x in dep_srv["_deps"]): continue new_deps = rec_deps(services, dep_name, start_point) deps.update(new_deps) From 3729b3bedbf379a9967c9e717dc17a337258e4ee Mon Sep 17 00:00:00 2001 From: Felix Rubio Date: Fri, 29 Nov 2024 21:14:13 +0100 Subject: [PATCH 7/8] Added missed update to deps set. Signed-off-by: Felix Rubio --- podman_compose.py | 1 + 1 file changed, 1 insertion(+) diff --git a/podman_compose.py b/podman_compose.py index d03b829d..a102a115 100755 --- a/podman_compose.py +++ b/podman_compose.py @@ -1348,6 +1348,7 @@ def rec_deps(services, service_name, start_point=None): deps_ls = [ServiceDependency(k, v.get("condition")) for k, v in deps_ls.items()] else: raise RuntimeError("depends_on should be a string, a list of strings or a dict") + deps.update(deps_ls) # parse link to get service name and remove alias links_ls = srv.get("links", []) From e6df1100f6f1b5338f7e4fbb52c73a8d45660a20 Mon Sep 17 00:00:00 2001 From: Felix Rubio Date: Fri, 29 Nov 2024 21:20:55 +0100 Subject: [PATCH 8/8] Handling dependencies comming from links. Signed-off-by: Felix Rubio --- podman_compose.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/podman_compose.py b/podman_compose.py index a102a115..a9c23ae5 100755 --- a/podman_compose.py +++ b/podman_compose.py @@ -1354,7 +1354,7 @@ def rec_deps(services, service_name, start_point=None): links_ls = srv.get("links", []) if not is_list(links_ls): links_ls = [links_ls] - deps.update([(c.split(":")[0] if ":" in c else c) for c in links_ls]) + deps.update([ServiceDependency(c.split(":")[0] if ":" in c else c) for c in links_ls]) for c in links_ls: if ":" in c: dep_name, dep_alias = c.split(":")