feat: switch brencher to dedicated connectivity tester #83
@@ -7,11 +7,13 @@ from steps.docker import DockerSwarmCheck, DockerSwarmDeploy
|
||||
from steps.git import CheckoutMerged, GitClone, GitUnmerge
|
||||
from steps.shared_state import SharedStateHolderInMemory
|
||||
|
||||
clone = GitClone(remotes=[
|
||||
{"name": "gitea_hermes", "url": "https://gitea.rudolf.keenetic.link/hermes/ansible"},
|
||||
{"name": "gitea_admin", "url": "https://gitea.rudolf.keenetic.link/admin/ansible"},
|
||||
],
|
||||
credEnvPrefix="GITEA",
|
||||
clone = GitClone(
|
||||
remotes=[
|
||||
{"name": "gitea_hermes", "url": "https://gitea.rudolf.keenetic.link/hermes/swarm_connectivity_tester"},
|
||||
{"name": "gitea_admin", "url": "https://gitea.rudolf.keenetic.link/admin/swarm_connectivity_tester"},
|
||||
{"name": "gitea_org1", "url": "https://gitea.rudolf.keenetic.link/org1/swarm_connectivity_tester"},
|
||||
],
|
||||
credEnvPrefix="GITEA",
|
||||
)
|
||||
|
||||
dockerSwarmCheck = DockerSwarmCheck(stack_name="connectivity_tester")
|
||||
@@ -22,7 +24,7 @@ state = SharedStateHolderInMemory(unmerge=unmerge)
|
||||
checkoutMerged = CheckoutMerged(
|
||||
clone,
|
||||
desired_branches=state,
|
||||
push="gitea_admin",
|
||||
push="gitea_org1",
|
||||
git_user_email="rudolfss13@gmail.com",
|
||||
git_user_name="brencher_bot",
|
||||
)
|
||||
@@ -30,8 +32,8 @@ checkoutMerged = CheckoutMerged(
|
||||
GITEA_API = os.environ.get("GITEA_API", "https://gitea.rudolf.keenetic.link/api/v1")
|
||||
GITEA_TOKEN = os.environ.get("GITEA_TOKEN", "")
|
||||
|
||||
WORKFLOW_OWNER = "admin"
|
||||
WORKFLOW_REPO = "ansible"
|
||||
WORKFLOW_OWNER = "org1"
|
||||
WORKFLOW_REPO = "swarm_connectivity_tester"
|
||||
WORKFLOW_FILE = "release.yml"
|
||||
|
||||
def _inputs() -> dict:
|
||||
@@ -53,7 +55,7 @@ await_build = GiteaActionsAwait(
|
||||
owner=WORKFLOW_OWNER,
|
||||
repo=WORKFLOW_REPO,
|
||||
token=GITEA_TOKEN,
|
||||
run=lambda: trigger_build.progress(),
|
||||
run=trigger_build,
|
||||
)
|
||||
|
||||
deployDocker = DockerSwarmDeploy(
|
||||
@@ -62,10 +64,10 @@ deployDocker = DockerSwarmDeploy(
|
||||
stackChecker=dockerSwarmCheck,
|
||||
envs=lambda: {
|
||||
"version": "auto-" + checkoutMerged.progress().version,
|
||||
"registry_owner": "admin",
|
||||
"registry_owner": "org1",
|
||||
},
|
||||
stack_name="connectivity_tester",
|
||||
docker_compose_path="connectivity_tester/stack-compose.yml",
|
||||
docker_compose_path="stack-compose.yml",
|
||||
)
|
||||
|
||||
logUrls = SimpleLog(message={
|
||||
|
||||
+63
-25
@@ -51,6 +51,8 @@ class GiteaActionsTrigger(AbstractStep[GiteaActionsRun]):
|
||||
self.wd = wd
|
||||
self.poll_timeout_sec = poll_timeout_sec
|
||||
self.poll_interval_sec = poll_interval_sec
|
||||
self._last_request_key: tuple[Optional[str], tuple[tuple[str, str], ...]] | None = None
|
||||
self._last_run: GiteaActionsRun | None = None
|
||||
|
||||
def _http(self, method: str, url: str, body: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
||||
data = None
|
||||
@@ -84,13 +86,40 @@ class GiteaActionsTrigger(AbstractStep[GiteaActionsRun]):
|
||||
)
|
||||
|
||||
def progress(self) -> GiteaActionsRun:
|
||||
latest = self._list_recent_runs()
|
||||
if self.env.dry:
|
||||
logger.info("DRY RUN: Skipping Gitea Actions workflow dispatch.")
|
||||
return GiteaActionsRun(
|
||||
run_id=0,
|
||||
status="dry-run",
|
||||
conclusion=None,
|
||||
html_url="",
|
||||
)
|
||||
|
||||
resolved_inputs: Dict[str, str] = {}
|
||||
if callable(self.inputs):
|
||||
resolved_inputs = self.inputs() or {}
|
||||
elif isinstance(self.inputs, dict):
|
||||
resolved_inputs = self.inputs
|
||||
ref = self.ref() if callable(self.ref) else self.ref
|
||||
request_key = (ref, tuple(sorted(resolved_inputs.items())))
|
||||
latest = self._list_recent_runs()
|
||||
|
||||
if latest is not None and latest.status != "completed":
|
||||
logger.info("Gitea Actions run %s is already active; reusing it.", latest.run_id)
|
||||
self._last_request_key = request_key
|
||||
self._last_run = latest
|
||||
return latest
|
||||
|
||||
if (
|
||||
self._last_request_key == request_key
|
||||
and self._last_run is not None
|
||||
and latest is not None
|
||||
and latest.run_id == self._last_run.run_id
|
||||
and latest.conclusion == "success"
|
||||
):
|
||||
logger.info("Gitea Actions run %s already succeeded; reusing it.", latest.run_id)
|
||||
return latest
|
||||
|
||||
body: Dict[str, Any] = {"ref": ref}
|
||||
if resolved_inputs:
|
||||
body["inputs"] = resolved_inputs
|
||||
@@ -116,6 +145,8 @@ class GiteaActionsTrigger(AbstractStep[GiteaActionsRun]):
|
||||
if current is None or (latest is not None and current.run_id == latest.run_id):
|
||||
time.sleep(self.poll_interval_sec)
|
||||
continue
|
||||
self._last_request_key = request_key
|
||||
self._last_run = current
|
||||
return current
|
||||
raise TimeoutError(
|
||||
f"Gitea Actions did not pick up a new run within {self.poll_timeout_sec}s"
|
||||
@@ -130,7 +161,7 @@ class GiteaActionsAwait(AbstractStep[GiteaActionsRun]):
|
||||
owner: str,
|
||||
repo: str,
|
||||
token: str,
|
||||
run: Callable[[], GiteaActionsRun],
|
||||
run: Union[AbstractStep[GiteaActionsRun], Callable[[], GiteaActionsRun]],
|
||||
poll_timeout_sec: int = 1800,
|
||||
poll_interval_sec: int = 10,
|
||||
**kwargs: Any) -> None:
|
||||
@@ -150,28 +181,35 @@ class GiteaActionsAwait(AbstractStep[GiteaActionsRun]):
|
||||
return json.loads(r.read().decode())
|
||||
|
||||
def progress(self) -> GiteaActionsRun:
|
||||
run = self._run()
|
||||
deadline = time.time() + self.poll_timeout_sec
|
||||
while time.time() < deadline:
|
||||
data = self._http(
|
||||
"GET",
|
||||
f"{self.api_base}/repos/{self.owner}/{self.repo}/actions/runs/{run.run_id}",
|
||||
if self.env.dry:
|
||||
logger.info("DRY RUN: Skipping wait for Gitea Actions workflow.")
|
||||
return GiteaActionsRun(
|
||||
run_id=0,
|
||||
status="dry-run",
|
||||
conclusion=None,
|
||||
html_url="",
|
||||
)
|
||||
status = data.get("status", "")
|
||||
conclusion = data.get("conclusion")
|
||||
if status == "completed":
|
||||
if conclusion != "success":
|
||||
raise RuntimeError(
|
||||
f"Gitea Actions run {run.run_id} concluded with {conclusion}: "
|
||||
f"{run.html_url}"
|
||||
)
|
||||
return GiteaActionsRun(
|
||||
run_id=run.run_id,
|
||||
status=status,
|
||||
conclusion=conclusion,
|
||||
html_url=run.html_url,
|
||||
)
|
||||
time.sleep(self.poll_interval_sec)
|
||||
raise TimeoutError(
|
||||
f"Gitea Actions run {run.run_id} did not complete within {self.poll_timeout_sec}s"
|
||||
|
||||
run = self._run.progress() if isinstance(self._run, AbstractStep) else self._run()
|
||||
data = self._http(
|
||||
"GET",
|
||||
f"{self.api_base}/repos/{self.owner}/{self.repo}/actions/runs/{run.run_id}",
|
||||
)
|
||||
status = data.get("status", "")
|
||||
conclusion = data.get("conclusion")
|
||||
if status != "completed":
|
||||
raise RuntimeError(
|
||||
f"Gitea Actions run {run.run_id} is not completed: {status}: "
|
||||
f"{run.html_url}"
|
||||
)
|
||||
if conclusion != "success":
|
||||
raise RuntimeError(
|
||||
f"Gitea Actions run {run.run_id} concluded with {conclusion}: "
|
||||
f"{run.html_url}"
|
||||
)
|
||||
return GiteaActionsRun(
|
||||
run_id=run.run_id,
|
||||
status=status,
|
||||
conclusion=conclusion,
|
||||
html_url=run.html_url,
|
||||
)
|
||||
|
||||
@@ -296,6 +296,7 @@ class DockerSwarmDeploy(AbstractStep[str]):
|
||||
# Deploy stack to Docker Swarm
|
||||
cmd = [
|
||||
"docker", "stack", "deploy",
|
||||
"--with-registry-auth",
|
||||
"--detach=false",
|
||||
"--prune",
|
||||
"-c", tmp_compose_path,
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
from typing import Any, Callable, Dict
|
||||
|
||||
import yaml
|
||||
from enironment import AbstractStep
|
||||
from steps.git import GitClone
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DockerComposePull(AbstractStep[Dict[str, str]]):
|
||||
"""Pull images referenced by a docker-compose file from a remote registry."""
|
||||
|
||||
def __init__(self,
|
||||
wd: GitClone,
|
||||
docker_compose_path: str,
|
||||
envs: Callable[[], Dict[str, Any]],
|
||||
**kwargs: Any) -> None:
|
||||
super().__init__(**kwargs)
|
||||
self.wd = wd
|
||||
self.envs = envs
|
||||
self.docker_compose_path = docker_compose_path
|
||||
|
||||
def progress(self) -> Dict[str, str]:
|
||||
env = self.envs()
|
||||
compose_path = os.path.join(self.wd.progress(), self.docker_compose_path)
|
||||
with open(compose_path, "r") as f:
|
||||
content = f.read()
|
||||
content = re.sub(r"\$\{([^}]+)\}", lambda m: env.get(m.group(1), ""), content)
|
||||
compose = yaml.safe_load(content)
|
||||
for svc in compose.get("services", {}).values():
|
||||
if "build" in svc:
|
||||
del svc["build"]
|
||||
tmp_path = compose_path + ".pull.yml"
|
||||
with open(tmp_path, "w") as f:
|
||||
yaml.safe_dump(compose, f)
|
||||
try:
|
||||
subprocess.run(
|
||||
["docker", "compose", "-f", tmp_path, "pull"],
|
||||
check=True,
|
||||
cwd=self.wd.progress(),
|
||||
)
|
||||
finally:
|
||||
os.unlink(tmp_path)
|
||||
return {name: svc.get("image", "") for name, svc in compose.get("services", {}).items()}
|
||||
@@ -0,0 +1,116 @@
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from enironment import AbstractStep, Environment
|
||||
from steps.actions import GiteaActionsAwait, GiteaActionsRun, GiteaActionsTrigger
|
||||
from steps.shared_state import SharedStateHolderInMemory
|
||||
|
||||
|
||||
class FixedRun(AbstractStep[GiteaActionsRun]):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.calls = 0
|
||||
|
||||
def progress(self) -> GiteaActionsRun:
|
||||
self.calls += 1
|
||||
return GiteaActionsRun(42, "queued", None, "http://example/run/42")
|
||||
|
||||
|
||||
def _environment(*steps: AbstractStep[Any], dry: bool) -> Environment:
|
||||
return Environment(
|
||||
id="actions-test",
|
||||
state=SharedStateHolderInMemory(unmerge=None),
|
||||
pipeline=list(steps),
|
||||
)
|
||||
|
||||
|
||||
def _trigger() -> GiteaActionsTrigger:
|
||||
return GiteaActionsTrigger(
|
||||
api_base="https://gitea.example/api/v1",
|
||||
owner="owner",
|
||||
repo="repo",
|
||||
workflow_file="release.yml",
|
||||
ref="main",
|
||||
token="token",
|
||||
)
|
||||
|
||||
|
||||
def test_trigger_skips_dispatch_in_dry_run() -> None:
|
||||
trigger = _trigger()
|
||||
env = _environment(trigger, dry=True)
|
||||
env.state.set_dry(True)
|
||||
trigger._http = lambda *args, **kwargs: (_ for _ in ()).throw(AssertionError("HTTP call")) # type: ignore[method-assign]
|
||||
|
||||
result = trigger.progress()
|
||||
|
||||
assert result.status == "dry-run"
|
||||
assert result.run_id == 0
|
||||
|
||||
|
||||
def test_trigger_reuses_active_run(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
trigger = _trigger()
|
||||
env = _environment(trigger, dry=False)
|
||||
runs = iter([
|
||||
None,
|
||||
GiteaActionsRun(42, "in_progress", None, "http://example/run/42"),
|
||||
GiteaActionsRun(42, "in_progress", None, "http://example/run/42"),
|
||||
])
|
||||
dispatches = 0
|
||||
|
||||
def fake_http(method: str, url: str, body: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
nonlocal dispatches
|
||||
if method == "POST":
|
||||
dispatches += 1
|
||||
return {}
|
||||
|
||||
monkeypatch.setattr(trigger, "_list_recent_runs", lambda: next(runs))
|
||||
monkeypatch.setattr(trigger, "_http", fake_http)
|
||||
trigger.poll_interval_sec = 0
|
||||
trigger.poll_timeout_sec = 1
|
||||
|
||||
first = trigger.progress()
|
||||
second = trigger.progress()
|
||||
|
||||
assert first.run_id == 42
|
||||
assert second.run_id == 42
|
||||
assert dispatches == 1
|
||||
|
||||
|
||||
def test_await_skips_polling_in_dry_run() -> None:
|
||||
run = FixedRun()
|
||||
await_step = GiteaActionsAwait(
|
||||
api_base="https://gitea.example/api/v1",
|
||||
owner="owner",
|
||||
repo="repo",
|
||||
token="token",
|
||||
run=run,
|
||||
)
|
||||
env = _environment(run, await_step, dry=True)
|
||||
env.state.set_dry(True)
|
||||
await_step._http = lambda *args, **kwargs: (_ for _ in ()).throw(AssertionError("HTTP call")) # type: ignore[method-assign]
|
||||
|
||||
result = await_step.progress()
|
||||
|
||||
assert result.status == "dry-run"
|
||||
assert run.calls == 0
|
||||
|
||||
|
||||
def test_await_raises_when_run_is_not_completed(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
run = FixedRun()
|
||||
await_step = GiteaActionsAwait(
|
||||
api_base="https://gitea.example/api/v1",
|
||||
owner="owner",
|
||||
repo="repo",
|
||||
token="token",
|
||||
run=run,
|
||||
)
|
||||
env = _environment(run, await_step, dry=False)
|
||||
monkeypatch.setattr(
|
||||
await_step,
|
||||
"_http",
|
||||
lambda *args, **kwargs: {"status": "in_progress", "conclusion": None},
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="is not completed"):
|
||||
await_step.progress()
|
||||
Reference in New Issue
Block a user