feat: switch brencher to dedicated connectivity tester #83

Open
hermes wants to merge 34 commits from hermes/brencher:feat/connectivity-tester-config into main
5 changed files with 242 additions and 36 deletions
+13 -11
View File
@@ -7,11 +7,13 @@ from steps.docker import DockerSwarmCheck, DockerSwarmDeploy
from steps.git import CheckoutMerged, GitClone, GitUnmerge
from steps.shared_state import SharedStateHolderInMemory
clone = GitClone(remotes=[
{"name": "gitea_hermes", "url": "https://gitea.rudolf.keenetic.link/hermes/ansible"},
{"name": "gitea_admin", "url": "https://gitea.rudolf.keenetic.link/admin/ansible"},
],
credEnvPrefix="GITEA",
clone = GitClone(
remotes=[
{"name": "gitea_hermes", "url": "https://gitea.rudolf.keenetic.link/hermes/swarm_connectivity_tester"},
{"name": "gitea_admin", "url": "https://gitea.rudolf.keenetic.link/admin/swarm_connectivity_tester"},
{"name": "gitea_org1", "url": "https://gitea.rudolf.keenetic.link/org1/swarm_connectivity_tester"},
],
credEnvPrefix="GITEA",
)
dockerSwarmCheck = DockerSwarmCheck(stack_name="connectivity_tester")
@@ -22,7 +24,7 @@ state = SharedStateHolderInMemory(unmerge=unmerge)
checkoutMerged = CheckoutMerged(
clone,
desired_branches=state,
push="gitea_admin",
push="gitea_org1",
git_user_email="rudolfss13@gmail.com",
git_user_name="brencher_bot",
)
@@ -30,8 +32,8 @@ checkoutMerged = CheckoutMerged(
GITEA_API = os.environ.get("GITEA_API", "https://gitea.rudolf.keenetic.link/api/v1")
GITEA_TOKEN = os.environ.get("GITEA_TOKEN", "")
WORKFLOW_OWNER = "admin"
WORKFLOW_REPO = "ansible"
WORKFLOW_OWNER = "org1"
WORKFLOW_REPO = "swarm_connectivity_tester"
WORKFLOW_FILE = "release.yml"
def _inputs() -> dict:
@@ -53,7 +55,7 @@ await_build = GiteaActionsAwait(
owner=WORKFLOW_OWNER,
repo=WORKFLOW_REPO,
token=GITEA_TOKEN,
run=lambda: trigger_build.progress(),
run=trigger_build,
)
deployDocker = DockerSwarmDeploy(
@@ -62,10 +64,10 @@ deployDocker = DockerSwarmDeploy(
stackChecker=dockerSwarmCheck,
envs=lambda: {
"version": "auto-" + checkoutMerged.progress().version,
"registry_owner": "admin",
"registry_owner": "org1",
},
stack_name="connectivity_tester",
docker_compose_path="connectivity_tester/stack-compose.yml",
docker_compose_path="stack-compose.yml",
)
logUrls = SimpleLog(message={
+63 -25
View File
@@ -51,6 +51,8 @@ class GiteaActionsTrigger(AbstractStep[GiteaActionsRun]):
self.wd = wd
self.poll_timeout_sec = poll_timeout_sec
self.poll_interval_sec = poll_interval_sec
self._last_request_key: tuple[Optional[str], tuple[tuple[str, str], ...]] | None = None
self._last_run: GiteaActionsRun | None = None
def _http(self, method: str, url: str, body: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
data = None
@@ -84,13 +86,40 @@ class GiteaActionsTrigger(AbstractStep[GiteaActionsRun]):
)
def progress(self) -> GiteaActionsRun:
latest = self._list_recent_runs()
if self.env.dry:
logger.info("DRY RUN: Skipping Gitea Actions workflow dispatch.")
return GiteaActionsRun(
run_id=0,
status="dry-run",
conclusion=None,
html_url="",
)
resolved_inputs: Dict[str, str] = {}
if callable(self.inputs):
resolved_inputs = self.inputs() or {}
elif isinstance(self.inputs, dict):
resolved_inputs = self.inputs
ref = self.ref() if callable(self.ref) else self.ref
request_key = (ref, tuple(sorted(resolved_inputs.items())))
latest = self._list_recent_runs()
if latest is not None and latest.status != "completed":
logger.info("Gitea Actions run %s is already active; reusing it.", latest.run_id)
self._last_request_key = request_key
self._last_run = latest
return latest
if (
self._last_request_key == request_key
and self._last_run is not None
and latest is not None
and latest.run_id == self._last_run.run_id
and latest.conclusion == "success"
):
logger.info("Gitea Actions run %s already succeeded; reusing it.", latest.run_id)
return latest
body: Dict[str, Any] = {"ref": ref}
if resolved_inputs:
body["inputs"] = resolved_inputs
@@ -116,6 +145,8 @@ class GiteaActionsTrigger(AbstractStep[GiteaActionsRun]):
if current is None or (latest is not None and current.run_id == latest.run_id):
time.sleep(self.poll_interval_sec)
continue
self._last_request_key = request_key
self._last_run = current
return current
raise TimeoutError(
f"Gitea Actions did not pick up a new run within {self.poll_timeout_sec}s"
@@ -130,7 +161,7 @@ class GiteaActionsAwait(AbstractStep[GiteaActionsRun]):
owner: str,
repo: str,
token: str,
run: Callable[[], GiteaActionsRun],
run: Union[AbstractStep[GiteaActionsRun], Callable[[], GiteaActionsRun]],
poll_timeout_sec: int = 1800,
poll_interval_sec: int = 10,
**kwargs: Any) -> None:
@@ -150,28 +181,35 @@ class GiteaActionsAwait(AbstractStep[GiteaActionsRun]):
return json.loads(r.read().decode())
def progress(self) -> GiteaActionsRun:
run = self._run()
deadline = time.time() + self.poll_timeout_sec
while time.time() < deadline:
data = self._http(
"GET",
f"{self.api_base}/repos/{self.owner}/{self.repo}/actions/runs/{run.run_id}",
if self.env.dry:
logger.info("DRY RUN: Skipping wait for Gitea Actions workflow.")
return GiteaActionsRun(
run_id=0,
status="dry-run",
conclusion=None,
html_url="",
)
status = data.get("status", "")
conclusion = data.get("conclusion")
if status == "completed":
if conclusion != "success":
raise RuntimeError(
f"Gitea Actions run {run.run_id} concluded with {conclusion}: "
f"{run.html_url}"
)
return GiteaActionsRun(
run_id=run.run_id,
status=status,
conclusion=conclusion,
html_url=run.html_url,
)
time.sleep(self.poll_interval_sec)
raise TimeoutError(
f"Gitea Actions run {run.run_id} did not complete within {self.poll_timeout_sec}s"
run = self._run.progress() if isinstance(self._run, AbstractStep) else self._run()
data = self._http(
"GET",
f"{self.api_base}/repos/{self.owner}/{self.repo}/actions/runs/{run.run_id}",
)
status = data.get("status", "")
conclusion = data.get("conclusion")
if status != "completed":
raise RuntimeError(
f"Gitea Actions run {run.run_id} is not completed: {status}: "
f"{run.html_url}"
)
if conclusion != "success":
raise RuntimeError(
f"Gitea Actions run {run.run_id} concluded with {conclusion}: "
f"{run.html_url}"
)
return GiteaActionsRun(
run_id=run.run_id,
status=status,
conclusion=conclusion,
html_url=run.html_url,
)
+1
View File
@@ -296,6 +296,7 @@ class DockerSwarmDeploy(AbstractStep[str]):
# Deploy stack to Docker Swarm
cmd = [
"docker", "stack", "deploy",
"--with-registry-auth",
"--detach=false",
"--prune",
"-c", tmp_compose_path,
+49
View File
@@ -0,0 +1,49 @@
import json
import logging
import os
import re
import subprocess
from typing import Any, Callable, Dict
import yaml
from enironment import AbstractStep
from steps.git import GitClone
logger = logging.getLogger(__name__)
class DockerComposePull(AbstractStep[Dict[str, str]]):
"""Pull images referenced by a docker-compose file from a remote registry."""
def __init__(self,
wd: GitClone,
docker_compose_path: str,
envs: Callable[[], Dict[str, Any]],
**kwargs: Any) -> None:
super().__init__(**kwargs)
self.wd = wd
self.envs = envs
self.docker_compose_path = docker_compose_path
def progress(self) -> Dict[str, str]:
env = self.envs()
compose_path = os.path.join(self.wd.progress(), self.docker_compose_path)
with open(compose_path, "r") as f:
content = f.read()
content = re.sub(r"\$\{([^}]+)\}", lambda m: env.get(m.group(1), ""), content)
compose = yaml.safe_load(content)
for svc in compose.get("services", {}).values():
if "build" in svc:
del svc["build"]
tmp_path = compose_path + ".pull.yml"
with open(tmp_path, "w") as f:
yaml.safe_dump(compose, f)
try:
subprocess.run(
["docker", "compose", "-f", tmp_path, "pull"],
check=True,
cwd=self.wd.progress(),
)
finally:
os.unlink(tmp_path)
return {name: svc.get("image", "") for name, svc in compose.get("services", {}).items()}
+116
View File
@@ -0,0 +1,116 @@
from typing import Any
import pytest
from enironment import AbstractStep, Environment
from steps.actions import GiteaActionsAwait, GiteaActionsRun, GiteaActionsTrigger
from steps.shared_state import SharedStateHolderInMemory
class FixedRun(AbstractStep[GiteaActionsRun]):
def __init__(self) -> None:
super().__init__()
self.calls = 0
def progress(self) -> GiteaActionsRun:
self.calls += 1
return GiteaActionsRun(42, "queued", None, "http://example/run/42")
def _environment(*steps: AbstractStep[Any], dry: bool) -> Environment:
return Environment(
id="actions-test",
state=SharedStateHolderInMemory(unmerge=None),
pipeline=list(steps),
)
def _trigger() -> GiteaActionsTrigger:
return GiteaActionsTrigger(
api_base="https://gitea.example/api/v1",
owner="owner",
repo="repo",
workflow_file="release.yml",
ref="main",
token="token",
)
def test_trigger_skips_dispatch_in_dry_run() -> None:
trigger = _trigger()
env = _environment(trigger, dry=True)
env.state.set_dry(True)
trigger._http = lambda *args, **kwargs: (_ for _ in ()).throw(AssertionError("HTTP call")) # type: ignore[method-assign]
result = trigger.progress()
assert result.status == "dry-run"
assert result.run_id == 0
def test_trigger_reuses_active_run(monkeypatch: pytest.MonkeyPatch) -> None:
trigger = _trigger()
env = _environment(trigger, dry=False)
runs = iter([
None,
GiteaActionsRun(42, "in_progress", None, "http://example/run/42"),
GiteaActionsRun(42, "in_progress", None, "http://example/run/42"),
])
dispatches = 0
def fake_http(method: str, url: str, body: dict[str, Any] | None = None) -> dict[str, Any]:
nonlocal dispatches
if method == "POST":
dispatches += 1
return {}
monkeypatch.setattr(trigger, "_list_recent_runs", lambda: next(runs))
monkeypatch.setattr(trigger, "_http", fake_http)
trigger.poll_interval_sec = 0
trigger.poll_timeout_sec = 1
first = trigger.progress()
second = trigger.progress()
assert first.run_id == 42
assert second.run_id == 42
assert dispatches == 1
def test_await_skips_polling_in_dry_run() -> None:
run = FixedRun()
await_step = GiteaActionsAwait(
api_base="https://gitea.example/api/v1",
owner="owner",
repo="repo",
token="token",
run=run,
)
env = _environment(run, await_step, dry=True)
env.state.set_dry(True)
await_step._http = lambda *args, **kwargs: (_ for _ in ()).throw(AssertionError("HTTP call")) # type: ignore[method-assign]
result = await_step.progress()
assert result.status == "dry-run"
assert run.calls == 0
def test_await_raises_when_run_is_not_completed(monkeypatch: pytest.MonkeyPatch) -> None:
run = FixedRun()
await_step = GiteaActionsAwait(
api_base="https://gitea.example/api/v1",
owner="owner",
repo="repo",
token="token",
run=run,
)
env = _environment(run, await_step, dry=False)
monkeypatch.setattr(
await_step,
"_http",
lambda *args, **kwargs: {"status": "in_progress", "conclusion": None},
)
with pytest.raises(RuntimeError, match="is not completed"):
await_step.progress()