diff --git a/backend/configs/connectivity_tester.py b/backend/configs/connectivity_tester.py index 05d1426..c972749 100644 --- a/backend/configs/connectivity_tester.py +++ b/backend/configs/connectivity_tester.py @@ -7,11 +7,13 @@ from steps.docker import DockerSwarmCheck, DockerSwarmDeploy from steps.git import CheckoutMerged, GitClone, GitUnmerge from steps.shared_state import SharedStateHolderInMemory -clone = GitClone(remotes=[ - {"name": "gitea_hermes", "url": "https://gitea.rudolf.keenetic.link/hermes/ansible"}, - {"name": "gitea_admin", "url": "https://gitea.rudolf.keenetic.link/admin/ansible"}, -], -credEnvPrefix="GITEA", +clone = GitClone( + remotes=[ + {"name": "gitea_hermes", "url": "https://gitea.rudolf.keenetic.link/hermes/swarm_connectivity_tester"}, + {"name": "gitea_admin", "url": "https://gitea.rudolf.keenetic.link/admin/swarm_connectivity_tester"}, + {"name": "gitea_org1", "url": "https://gitea.rudolf.keenetic.link/org1/swarm_connectivity_tester"}, + ], + credEnvPrefix="GITEA", ) dockerSwarmCheck = DockerSwarmCheck(stack_name="connectivity_tester") @@ -22,7 +24,7 @@ state = SharedStateHolderInMemory(unmerge=unmerge) checkoutMerged = CheckoutMerged( clone, desired_branches=state, - push="gitea_admin", + push="gitea_org1", git_user_email="rudolfss13@gmail.com", git_user_name="brencher_bot", ) @@ -30,8 +32,8 @@ checkoutMerged = CheckoutMerged( GITEA_API = os.environ.get("GITEA_API", "https://gitea.rudolf.keenetic.link/api/v1") GITEA_TOKEN = os.environ.get("GITEA_TOKEN", "") -WORKFLOW_OWNER = "admin" -WORKFLOW_REPO = "ansible" +WORKFLOW_OWNER = "org1" +WORKFLOW_REPO = "swarm_connectivity_tester" WORKFLOW_FILE = "release.yml" def _inputs() -> dict: @@ -53,7 +55,7 @@ await_build = GiteaActionsAwait( owner=WORKFLOW_OWNER, repo=WORKFLOW_REPO, token=GITEA_TOKEN, - run=lambda: trigger_build.progress(), + run=trigger_build, ) deployDocker = DockerSwarmDeploy( @@ -62,10 +64,10 @@ deployDocker = DockerSwarmDeploy( stackChecker=dockerSwarmCheck, envs=lambda: { "version": "auto-" + checkoutMerged.progress().version, - "registry_owner": "admin", + "registry_owner": "org1", }, stack_name="connectivity_tester", - docker_compose_path="connectivity_tester/stack-compose.yml", + docker_compose_path="stack-compose.yml", ) logUrls = SimpleLog(message={ diff --git a/backend/steps/actions.py b/backend/steps/actions.py index c805256..3d57c60 100644 --- a/backend/steps/actions.py +++ b/backend/steps/actions.py @@ -51,6 +51,8 @@ class GiteaActionsTrigger(AbstractStep[GiteaActionsRun]): self.wd = wd self.poll_timeout_sec = poll_timeout_sec self.poll_interval_sec = poll_interval_sec + self._last_request_key: tuple[Optional[str], tuple[tuple[str, str], ...]] | None = None + self._last_run: GiteaActionsRun | None = None def _http(self, method: str, url: str, body: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: data = None @@ -84,13 +86,40 @@ class GiteaActionsTrigger(AbstractStep[GiteaActionsRun]): ) def progress(self) -> GiteaActionsRun: - latest = self._list_recent_runs() + if self.env.dry: + logger.info("DRY RUN: Skipping Gitea Actions workflow dispatch.") + return GiteaActionsRun( + run_id=0, + status="dry-run", + conclusion=None, + html_url="", + ) + resolved_inputs: Dict[str, str] = {} if callable(self.inputs): resolved_inputs = self.inputs() or {} elif isinstance(self.inputs, dict): resolved_inputs = self.inputs ref = self.ref() if callable(self.ref) else self.ref + request_key = (ref, tuple(sorted(resolved_inputs.items()))) + latest = self._list_recent_runs() + + if latest is not None and latest.status != "completed": + logger.info("Gitea Actions run %s is already active; reusing it.", latest.run_id) + self._last_request_key = request_key + self._last_run = latest + return latest + + if ( + self._last_request_key == request_key + and self._last_run is not None + and latest is not None + and latest.run_id == self._last_run.run_id + and latest.conclusion == "success" + ): + logger.info("Gitea Actions run %s already succeeded; reusing it.", latest.run_id) + return latest + body: Dict[str, Any] = {"ref": ref} if resolved_inputs: body["inputs"] = resolved_inputs @@ -116,6 +145,8 @@ class GiteaActionsTrigger(AbstractStep[GiteaActionsRun]): if current is None or (latest is not None and current.run_id == latest.run_id): time.sleep(self.poll_interval_sec) continue + self._last_request_key = request_key + self._last_run = current return current raise TimeoutError( f"Gitea Actions did not pick up a new run within {self.poll_timeout_sec}s" @@ -130,7 +161,7 @@ class GiteaActionsAwait(AbstractStep[GiteaActionsRun]): owner: str, repo: str, token: str, - run: Callable[[], GiteaActionsRun], + run: Union[AbstractStep[GiteaActionsRun], Callable[[], GiteaActionsRun]], poll_timeout_sec: int = 1800, poll_interval_sec: int = 10, **kwargs: Any) -> None: @@ -150,28 +181,35 @@ class GiteaActionsAwait(AbstractStep[GiteaActionsRun]): return json.loads(r.read().decode()) def progress(self) -> GiteaActionsRun: - run = self._run() - deadline = time.time() + self.poll_timeout_sec - while time.time() < deadline: - data = self._http( - "GET", - f"{self.api_base}/repos/{self.owner}/{self.repo}/actions/runs/{run.run_id}", + if self.env.dry: + logger.info("DRY RUN: Skipping wait for Gitea Actions workflow.") + return GiteaActionsRun( + run_id=0, + status="dry-run", + conclusion=None, + html_url="", ) - status = data.get("status", "") - conclusion = data.get("conclusion") - if status == "completed": - if conclusion != "success": - raise RuntimeError( - f"Gitea Actions run {run.run_id} concluded with {conclusion}: " - f"{run.html_url}" - ) - return GiteaActionsRun( - run_id=run.run_id, - status=status, - conclusion=conclusion, - html_url=run.html_url, - ) - time.sleep(self.poll_interval_sec) - raise TimeoutError( - f"Gitea Actions run {run.run_id} did not complete within {self.poll_timeout_sec}s" + + run = self._run.progress() if isinstance(self._run, AbstractStep) else self._run() + data = self._http( + "GET", + f"{self.api_base}/repos/{self.owner}/{self.repo}/actions/runs/{run.run_id}", + ) + status = data.get("status", "") + conclusion = data.get("conclusion") + if status != "completed": + raise RuntimeError( + f"Gitea Actions run {run.run_id} is not completed: {status}: " + f"{run.html_url}" + ) + if conclusion != "success": + raise RuntimeError( + f"Gitea Actions run {run.run_id} concluded with {conclusion}: " + f"{run.html_url}" + ) + return GiteaActionsRun( + run_id=run.run_id, + status=status, + conclusion=conclusion, + html_url=run.html_url, ) diff --git a/backend/steps/docker.py b/backend/steps/docker.py index 94712bc..942d6e9 100644 --- a/backend/steps/docker.py +++ b/backend/steps/docker.py @@ -296,6 +296,7 @@ class DockerSwarmDeploy(AbstractStep[str]): # Deploy stack to Docker Swarm cmd = [ "docker", "stack", "deploy", + "--with-registry-auth", "--detach=false", "--prune", "-c", tmp_compose_path, diff --git a/backend/steps/docker_pull.py b/backend/steps/docker_pull.py new file mode 100644 index 0000000..93e5120 --- /dev/null +++ b/backend/steps/docker_pull.py @@ -0,0 +1,49 @@ +import json +import logging +import os +import re +import subprocess +from typing import Any, Callable, Dict + +import yaml +from enironment import AbstractStep +from steps.git import GitClone + +logger = logging.getLogger(__name__) + + +class DockerComposePull(AbstractStep[Dict[str, str]]): + """Pull images referenced by a docker-compose file from a remote registry.""" + + def __init__(self, + wd: GitClone, + docker_compose_path: str, + envs: Callable[[], Dict[str, Any]], + **kwargs: Any) -> None: + super().__init__(**kwargs) + self.wd = wd + self.envs = envs + self.docker_compose_path = docker_compose_path + + def progress(self) -> Dict[str, str]: + env = self.envs() + compose_path = os.path.join(self.wd.progress(), self.docker_compose_path) + with open(compose_path, "r") as f: + content = f.read() + content = re.sub(r"\$\{([^}]+)\}", lambda m: env.get(m.group(1), ""), content) + compose = yaml.safe_load(content) + for svc in compose.get("services", {}).values(): + if "build" in svc: + del svc["build"] + tmp_path = compose_path + ".pull.yml" + with open(tmp_path, "w") as f: + yaml.safe_dump(compose, f) + try: + subprocess.run( + ["docker", "compose", "-f", tmp_path, "pull"], + check=True, + cwd=self.wd.progress(), + ) + finally: + os.unlink(tmp_path) + return {name: svc.get("image", "") for name, svc in compose.get("services", {}).items()} diff --git a/tests/test_actions.py b/tests/test_actions.py new file mode 100644 index 0000000..2038987 --- /dev/null +++ b/tests/test_actions.py @@ -0,0 +1,116 @@ +from typing import Any + +import pytest + +from enironment import AbstractStep, Environment +from steps.actions import GiteaActionsAwait, GiteaActionsRun, GiteaActionsTrigger +from steps.shared_state import SharedStateHolderInMemory + + +class FixedRun(AbstractStep[GiteaActionsRun]): + def __init__(self) -> None: + super().__init__() + self.calls = 0 + + def progress(self) -> GiteaActionsRun: + self.calls += 1 + return GiteaActionsRun(42, "queued", None, "http://example/run/42") + + +def _environment(*steps: AbstractStep[Any], dry: bool) -> Environment: + return Environment( + id="actions-test", + state=SharedStateHolderInMemory(unmerge=None), + pipeline=list(steps), + ) + + +def _trigger() -> GiteaActionsTrigger: + return GiteaActionsTrigger( + api_base="https://gitea.example/api/v1", + owner="owner", + repo="repo", + workflow_file="release.yml", + ref="main", + token="token", + ) + + +def test_trigger_skips_dispatch_in_dry_run() -> None: + trigger = _trigger() + env = _environment(trigger, dry=True) + env.state.set_dry(True) + trigger._http = lambda *args, **kwargs: (_ for _ in ()).throw(AssertionError("HTTP call")) # type: ignore[method-assign] + + result = trigger.progress() + + assert result.status == "dry-run" + assert result.run_id == 0 + + +def test_trigger_reuses_active_run(monkeypatch: pytest.MonkeyPatch) -> None: + trigger = _trigger() + env = _environment(trigger, dry=False) + runs = iter([ + None, + GiteaActionsRun(42, "in_progress", None, "http://example/run/42"), + GiteaActionsRun(42, "in_progress", None, "http://example/run/42"), + ]) + dispatches = 0 + + def fake_http(method: str, url: str, body: dict[str, Any] | None = None) -> dict[str, Any]: + nonlocal dispatches + if method == "POST": + dispatches += 1 + return {} + + monkeypatch.setattr(trigger, "_list_recent_runs", lambda: next(runs)) + monkeypatch.setattr(trigger, "_http", fake_http) + trigger.poll_interval_sec = 0 + trigger.poll_timeout_sec = 1 + + first = trigger.progress() + second = trigger.progress() + + assert first.run_id == 42 + assert second.run_id == 42 + assert dispatches == 1 + + +def test_await_skips_polling_in_dry_run() -> None: + run = FixedRun() + await_step = GiteaActionsAwait( + api_base="https://gitea.example/api/v1", + owner="owner", + repo="repo", + token="token", + run=run, + ) + env = _environment(run, await_step, dry=True) + env.state.set_dry(True) + await_step._http = lambda *args, **kwargs: (_ for _ in ()).throw(AssertionError("HTTP call")) # type: ignore[method-assign] + + result = await_step.progress() + + assert result.status == "dry-run" + assert run.calls == 0 + + +def test_await_raises_when_run_is_not_completed(monkeypatch: pytest.MonkeyPatch) -> None: + run = FixedRun() + await_step = GiteaActionsAwait( + api_base="https://gitea.example/api/v1", + owner="owner", + repo="repo", + token="token", + run=run, + ) + env = _environment(run, await_step, dry=False) + monkeypatch.setattr( + await_step, + "_http", + lambda *args, **kwargs: {"status": "in_progress", "conclusion": None}, + ) + + with pytest.raises(RuntimeError, match="is not completed"): + await_step.progress()