diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..4338684 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,111 @@ +# Adding a new stack to brencher + +A "stack" is one self-contained Docker Swarm service that brencher manages +end-to-end: detect HEAD changes, build, deploy, and expose service status to +the UI. This file documents how to wire one up so it picks up commits +without an operator clicking "Update" in the UI every time. + +## Repository layout + +Brencher looks at exactly one commit on one ref per environment. For a new +stack `my_stack`: + +1. Push branch `feat/my_stack` (or whatever the team convention is) to a + Git repo listed in `remotes` of the config file. +2. Drop the repo in `gitea` or `github` so it's reachable through the same + remotes that other stacks use. +3. Make sure `my_stack/stack-compose.yml` and a Dockerfile (typically + `my_stack/Dockerfile`) exist at the path the config will point at. + +## Config file: `backend/configs/.py` + +The shape is fixed by `enironment.Environment` and the steps it pulls in. +Look at `backend/configs/hermes.py` for the canonical pattern. A new config +must: + +- Build a `GitClone` with at least one remote whose `url` points to the repo + above. +- Build a `DockerSwarmCheck(stack_name="")` to introspect the + deployed service. +- Build **`GitUnmerge(clone, dockerSwarmCheck)`** and wrap it into + `SharedStateHolderInMemory(unmerge=unmerge)`. This is what makes + autodeploy work — `GitUnmerge` reads the `org.brencher.version` label + from the running swarm task and converts it into + `desired_branches`, so each polling cycle asks "is HEAD ≠ what's + deployed?" without any UI input. +- Wire a `CheckoutMerged(clone, desired_branches=state, push=False, …)`. + Push stays `False` for first-party stacks; set `True` only when you want + brencher to push auto-named branches back to origin. +- Build a `DockerComposeBuild` (or `DockerSwarmBuild` if a Swarmfile is + used) pointing at the compose path in the repo. +- Build a `DockerSwarmDeploy(wd=clone, buildDocker=…, stackChecker=…, …)` + with the same `stack_name`. Image tag is `auto-` driven by + `checkoutMerged.progress().version`. +- Optionally append a `SimpleLog(message={"userLinks": {...}})` for the + link to surface in the UI. +- Set `__all__` and export the `Environment` named after the stack id. +- Add it to `Environment.pipeline` in the canonical order: + `clone → state → checkoutMerged → dockerSwarmCheck → build → deploy → + optionals`. Build and deploy are explicit `DockerComposeBuild` and + `DockerSwarmDeploy` siblings; do not collapse them into one step. + +If you skip the `GitUnmerge` line, `state.branches` will stay empty and +the stack will only deploy when an operator pushes a `branches` update +through the UI. Don't skip it; it's the difference between a self-driving +stack and a manual one. See `connectivity_tester.py` history for the bug +this caused. + +## Verification checklist + +- `python3 -c "from backend.configs.my_stack import my_stack"` works in + the brencher container. +- `PROFILES="my_stack" python3 backend/app.py` boots and the UI renders a + row for `my_stack` with non-empty branches (from `GitUnmerge`, + pointing at whatever image is currently deployed). +- Push to the configured branch, wait one full polling cycle + (~`RESET_INTERVAL` = 180s, plus the next pipeline tick), then confirm + the service was redeployed with a new `org.brencher.version` label. +- No UI `environment_update` was issued in between. + +## Common foot-guns + +- `dockerfile:` path in `stack-compose.yml` is interpreted relative to the + compose file's directory. The compose file and the Dockerfile must + live side-by-side; use `dockerfile: Dockerfile`, not + `dockerfile: `. +- Removing the `name:` key from the top of `stack-compose.yml` is fine + for `docker stack deploy` — the stack name comes from the CLI option + brencher sets. +- `EXTRA_ENDPOINTS` and friends are env vars; if you reference them in + compose, define them in the `envs=lambda: {...}` of the + `DockerComposeBuild` and `DockerSwarmDeploy`. `substitute` uses + `env.get(...)` for deploys but `env[...]` (raises) for builds, so an + unlisted variable kills the build but quietly produces an empty string + in the deploy. +- Compose files must not assume Docker networks that don't exist. Use + either an `external: true` net that already exists in Swarm, drop + the `networks:` block entirely, or `driver: overlay` for a + stack-private net. + +## Tracking + +When you add a stack, also: + +- Open a tracking issue in `admin/brencher` titled + `[ready-to-test] : …` with `state=` from your + `GitUnmerge`-derived branches and what URL/port the service exposes. +- Add the stack id to the `PROFILES` env var in the brencher stack + compose (`docker_stack` / Swarm declarative) so it gets picked up on + next rollout. The admin overlay is where this lives. + +## Build delegation pattern (Actions) + +Brencher does not run docker build or docker push itself. It dispatches a Gitea +Actions workflow via `POST /api/v1/repos/{owner}/{repo}/actions/workflows/{file}/dispatches`, +waits for the run to complete, then runs `docker stack deploy`. + +Use `steps.actions.GiteaActionsTrigger` and `steps.actions.GiteaActionsAwait` +in the stack config, set `buildDocker=None` on `DockerSwarmDeploy`, and let Swarm +resolve and pull the image on target nodes during deploy. Credentials must be +available to the deploy path and the compose image must point to a registry the +runner can push to (e.g. Gitea Container Registry `gitea.rudolf.keenetic.link//`). diff --git a/README.instruction.md b/README.instruction.md index e48affa..883f4bc 100644 --- a/README.instruction.md +++ b/README.instruction.md @@ -33,7 +33,7 @@ docker service update --secret-rm brencher-secrets brencher_brencher-backend docker service update --secret-rm brencher-secrets brencher2_brencher-backend docker secret rm brencher-secrets -printf "GIT_USERNAME=git\nGIT_PASSWORD=TODO\n" | \ +printf "GIT_USERNAME=git\nGIT_PASSWORD=TODO\nGITEA_TOKEN=TODO\n" | \ docker secret create brencher-secrets - docker service update --secret-add brencher-secrets brencher_brencher-backend diff --git a/backend/configs/connectivity_tester.py b/backend/configs/connectivity_tester.py new file mode 100644 index 0000000..05d1426 --- /dev/null +++ b/backend/configs/connectivity_tester.py @@ -0,0 +1,91 @@ +import os + +from enironment import Environment +from steps.actions import GiteaActionsAwait, GiteaActionsTrigger +from steps.checks import SimpleLog +from steps.docker import DockerSwarmCheck, DockerSwarmDeploy +from steps.git import CheckoutMerged, GitClone, GitUnmerge +from steps.shared_state import SharedStateHolderInMemory + +clone = GitClone(remotes=[ + {"name": "gitea_hermes", "url": "https://gitea.rudolf.keenetic.link/hermes/ansible"}, + {"name": "gitea_admin", "url": "https://gitea.rudolf.keenetic.link/admin/ansible"}, +], +credEnvPrefix="GITEA", +) + +dockerSwarmCheck = DockerSwarmCheck(stack_name="connectivity_tester") + +unmerge = GitUnmerge(clone, dockerSwarmCheck) +state = SharedStateHolderInMemory(unmerge=unmerge) + +checkoutMerged = CheckoutMerged( + clone, + desired_branches=state, + push="gitea_admin", + git_user_email="rudolfss13@gmail.com", + git_user_name="brencher_bot", +) + +GITEA_API = os.environ.get("GITEA_API", "https://gitea.rudolf.keenetic.link/api/v1") +GITEA_TOKEN = os.environ.get("GITEA_TOKEN", "") + +WORKFLOW_OWNER = "admin" +WORKFLOW_REPO = "ansible" +WORKFLOW_FILE = "release.yml" + +def _inputs() -> dict: + return {"tag": "auto-" + checkoutMerged.progress().version} + +trigger_build = GiteaActionsTrigger( + api_base=GITEA_API, + owner=WORKFLOW_OWNER, + repo=WORKFLOW_REPO, + workflow_file=WORKFLOW_FILE, + ref=lambda: checkoutMerged.progress().remote_branch_name or "master", + token=GITEA_TOKEN, + inputs=_inputs, + wd=checkoutMerged, +) + +await_build = GiteaActionsAwait( + api_base=GITEA_API, + owner=WORKFLOW_OWNER, + repo=WORKFLOW_REPO, + token=GITEA_TOKEN, + run=lambda: trigger_build.progress(), +) + +deployDocker = DockerSwarmDeploy( + wd=clone, + buildDocker=None, + stackChecker=dockerSwarmCheck, + envs=lambda: { + "version": "auto-" + checkoutMerged.progress().version, + "registry_owner": "admin", + }, + stack_name="connectivity_tester", + docker_compose_path="connectivity_tester/stack-compose.yml", +) + +logUrls = SimpleLog(message={ + "userLinks": { + "Connectivity tester": "http://connectivity-tester.keenetic.link:8765/", + }, +}) + +__all__ = ["connectivity_tester"] +connectivity_tester = Environment( + id="connectivity_tester", + state=state, + pipeline=[ + clone, + state, + checkoutMerged, + dockerSwarmCheck, + trigger_build, + await_build, + deployDocker, + logUrls, + ], +) diff --git a/backend/steps/actions.py b/backend/steps/actions.py new file mode 100644 index 0000000..c805256 --- /dev/null +++ b/backend/steps/actions.py @@ -0,0 +1,177 @@ +import json +import logging +import os +import time +import urllib.error +import urllib.request +from dataclasses import dataclass +from typing import Any, Callable, Dict, Optional, Union + +from enironment import AbstractStep +from steps.git import CheckoutMerged + +logger = logging.getLogger(__name__) + + +@dataclass +class GiteaActionsRun: + run_id: int + status: str + conclusion: Optional[str] + html_url: str + + +InputsSource = Union[Dict[str, str], Callable[[], Dict[str, str]], None] +RefSource = Union[str, Callable[[], Optional[str]], None] + + +class GiteaActionsTrigger(AbstractStep[GiteaActionsRun]): + """Trigger a Gitea Actions workflow via the dispatches API.""" + + def __init__(self, + api_base: str, + owner: str, + repo: str, + workflow_file: str, + ref: RefSource, + token: str, + inputs: InputsSource = None, + wd: Optional[CheckoutMerged] = None, + poll_timeout_sec: int = 1800, + poll_interval_sec: int = 10, + **kwargs: Any) -> None: + super().__init__(**kwargs) + self.api_base = api_base.rstrip("/") + self.owner = owner + self.repo = repo + self.workflow_file = workflow_file + self.ref = ref + self.token = token + self.inputs = inputs + self.wd = wd + self.poll_timeout_sec = poll_timeout_sec + self.poll_interval_sec = poll_interval_sec + + def _http(self, method: str, url: str, body: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + data = None + headers = {"Authorization": f"token {self.token}", "Accept": "application/json"} + if body is not None: + data = json.dumps(body).encode() + headers["Content-Type"] = "application/json" + req = urllib.request.Request(url, data=data, headers=headers, method=method) + with urllib.request.urlopen(req, timeout=30) as r: + body_bytes = r.read() + if not body_bytes or r.status == 204: + return {} + return json.loads(body_bytes.decode()) + + def _list_recent_runs(self) -> Optional[GiteaActionsRun]: + url = f"{self.api_base}/repos/{self.owner}/{self.repo}/actions/runs?limit=1" + try: + data = self._http("GET", url) + except urllib.error.HTTPError as e: + logger.error("list runs failed: %s", e) + return None + items = data.get("workflow_runs") or [] + if not items: + return None + r = items[0] + return GiteaActionsRun( + run_id=r["id"], + status=r.get("status", ""), + conclusion=r.get("conclusion"), + html_url=r.get("html_url", ""), + ) + + def progress(self) -> GiteaActionsRun: + latest = self._list_recent_runs() + resolved_inputs: Dict[str, str] = {} + if callable(self.inputs): + resolved_inputs = self.inputs() or {} + elif isinstance(self.inputs, dict): + resolved_inputs = self.inputs + ref = self.ref() if callable(self.ref) else self.ref + body: Dict[str, Any] = {"ref": ref} + if resolved_inputs: + body["inputs"] = resolved_inputs + url = ( + f"{self.api_base}/repos/{self.owner}/{self.repo}" + f"/actions/workflows/{self.workflow_file}/dispatches" + ) + try: + self._http("POST", url, body=body) + except urllib.error.HTTPError as e: + if e.code != 500: + raise + raise RuntimeError( + "POST /dispatches returned 500 (gitea-side bug on this instance). " + "Workaround: trigger the workflow manually by pushing a tag matching " + f"the workflow's `on.push.tags` filter, e.g. " + f"`git push HEAD:refs/tags/auto-`. " + f"Requested workflow: {self.owner}/{self.repo}/{self.workflow_file}@{ref}" + ) from e + deadline = time.time() + self.poll_timeout_sec + while time.time() < deadline: + current = self._list_recent_runs() + if current is None or (latest is not None and current.run_id == latest.run_id): + time.sleep(self.poll_interval_sec) + continue + return current + raise TimeoutError( + f"Gitea Actions did not pick up a new run within {self.poll_timeout_sec}s" + ) + + +class GiteaActionsAwait(AbstractStep[GiteaActionsRun]): + """Wait for a previously-triggered Gitea Actions run to complete.""" + + def __init__(self, + api_base: str, + owner: str, + repo: str, + token: str, + run: Callable[[], GiteaActionsRun], + poll_timeout_sec: int = 1800, + poll_interval_sec: int = 10, + **kwargs: Any) -> None: + super().__init__(**kwargs) + self.api_base = api_base.rstrip("/") + self.owner = owner + self.repo = repo + self.token = token + self._run = run + self.poll_timeout_sec = poll_timeout_sec + self.poll_interval_sec = poll_interval_sec + + def _http(self, method: str, url: str) -> Dict[str, Any]: + headers = {"Authorization": f"token {self.token}", "Accept": "application/json"} + req = urllib.request.Request(url, headers=headers, method=method) + with urllib.request.urlopen(req, timeout=30) as r: + return json.loads(r.read().decode()) + + def progress(self) -> GiteaActionsRun: + run = self._run() + deadline = time.time() + self.poll_timeout_sec + while time.time() < deadline: + data = self._http( + "GET", + f"{self.api_base}/repos/{self.owner}/{self.repo}/actions/runs/{run.run_id}", + ) + status = data.get("status", "") + conclusion = data.get("conclusion") + if status == "completed": + if conclusion != "success": + raise RuntimeError( + f"Gitea Actions run {run.run_id} concluded with {conclusion}: " + f"{run.html_url}" + ) + return GiteaActionsRun( + run_id=run.run_id, + status=status, + conclusion=conclusion, + html_url=run.html_url, + ) + time.sleep(self.poll_interval_sec) + raise TimeoutError( + f"Gitea Actions run {run.run_id} did not complete within {self.poll_timeout_sec}s" + ) diff --git a/backend/steps/docker.py b/backend/steps/docker.py index 9c08d81..94712bc 100644 --- a/backend/steps/docker.py +++ b/backend/steps/docker.py @@ -230,6 +230,7 @@ class DockerSwarmDeploy(AbstractStep[str]): del svc["build"] svc["labels"] = {"org.brencher.version": env["version"]} del env["version"] + env.pop("registry_owner", None) merge_dicts(compose, env) # logger.info(f"Final compose: {compose}") content = yaml.safe_dump(compose) diff --git a/backend/steps/git.py b/backend/steps/git.py index 8df31d2..93ca504 100644 --- a/backend/steps/git.py +++ b/backend/steps/git.py @@ -103,13 +103,19 @@ class GitClone(AbstractStep[str]): remote.set_url(auth_url) else: remote = repo.create_remote(remote_name, auth_url) + with repo.config_writer() as config: + config.set_value( + f'remote "{remote_name}"', + 'fetch', + f'+refs/heads/*:refs/remotes/{remote_name}/*', + ) if remote_name == "origin" and self.branchNamePrefix != "": with repo.config_writer() as config: config.set_value('remote "origin"', "fetch", f"+refs/heads/{self.branchNamePrefix}/*:refs/remotes/origin/{self.branchNamePrefix}/*") result = remote.fetch(prune=True) if not result or any(fetch_info.flags & fetch_info.ERROR for fetch_info in result): - raise BaseException(f"Failed to fetch updates for {repository_url}") + raise BaseException(f"Failed to fetch updates for {remote_name}") if not os.path.exists(os.path.join(self.repo_path, ".git")): raise BaseException(f"Failed to clone repositories {self.remotes} to {self.repo_path}") except BaseException as e: @@ -183,7 +189,7 @@ class CheckoutMerged(AbstractStep[CheckoutAndMergeResult]): desired_branches: AbstractStep[SharedState], git_user_email: str, git_user_name: str, - push: bool = True, + push: bool | str = True, **kwargs: Any): super().__init__(**kwargs) self.wd = wd @@ -306,12 +312,13 @@ class CheckoutMerged(AbstractStep[CheckoutAndMergeResult]): remote_name, branch_name = ref.name.split('/', 1) remote_branch_name = branch_name if remote_name == 'origin' else ref.name break - if self.push and remote_branch_name is None: + if self.push: auto_branch_hash = hashlib.sha1(''.join([x.hexsha for x in sorted_commits]).encode()).hexdigest() auto_branch_name = f"auto-{version}" + push_remote = self.push if isinstance(self.push, str) else "origin" logger.info(f"Pushing {repo.head.commit.hexsha} -> {auto_branch_name}") - repo.git.push('-f', 'origin', f"HEAD:refs/heads/{auto_branch_name}") + repo.git.push('-f', push_remote, f"HEAD:refs/heads/{auto_branch_name}") remote_branch_name = auto_branch_name return CheckoutAndMergeResult( diff --git a/backend/web.py b/backend/web.py index 5f8ae4c..313f6e9 100644 --- a/backend/web.py +++ b/backend/web.py @@ -94,6 +94,15 @@ class WebApp: merged = _merge_dicts(merged, connector.branches) return merged + def get_selection_summary(self) -> Dict[str, Dict[str, Any]]: + return { + env_id: { + "branches": env.get("branches", []), + "dry": env.get("dry"), + } + for env_id, env in self.get_global_envs_to_emit().items() + } + # --- Async scheduling from sync threads --- def _schedule_async(self, coro: Any) -> None: @@ -164,6 +173,7 @@ class WebApp: async def websocket_endpoint(self, websocket: WebSocket) -> None: await websocket.accept() state = self.ws_connections.setdefault(websocket, {}) + logger.info("UI connected: selected=%s", self.get_selection_summary()) try: # Send initial state on connect and record it so duplicate broadcasts are suppressed @@ -182,7 +192,12 @@ class WebApp: if "update" in message: update_data = message.get("update") or {} - logger.info(f"Received environment update: {update_data}") + logger.info( + "UI selection update: env=%s branches=%s dry=%s", + update_data.get('id', ''), + update_data.get('branches'), + update_data.get('dry'), + ) if self.secondaryManager: await self.secondaryManager.send({"update": update_data}) id = update_data.get('id', '') @@ -222,6 +237,7 @@ class WebApp: if isinstance(p, CachingStep): p.reset() logger.info(f"Updated environment {env.id} branches to {update_data.get('branches')}, dry={update_data.get('dry')}") + logger.info("UI selection resolved: selected=%s", self.get_selection_summary()) self.core.environment_update_event.set() await self.broadcast_environments(self.get_global_envs_to_emit()) diff --git a/docker-compose.yml b/docker-compose.yml index b4154d2..7d59b94 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,7 +17,7 @@ services: - VIRTUAL_HOST=brencher.rudolf.keenetic.link - LETSENCRYPT_HOST=brencher.rudolf.keenetic.link - VERSION=${version} - - PROFILES=brencher2,torrserv_proxy,immich,gmail_mcp,hermes,hermes_infra,gitea + - PROFILES=brencher2,torrserv_proxy,immich,gmail_mcp,hermes,hermes_infra,gitea,connectivity_tester - SECONDARY_BRENCHER=http://host.docker.internal:5002 deploy: replicas: 1