From d95d97be68402dcd8556a2968decdce2868baf50 Mon Sep 17 00:00:00 2001 From: hermes Date: Mon, 14 Sep 2026 13:05:12 +0000 Subject: [PATCH 01/43] feat(configs): add connectivity_tester stack Long-running Swarm service that runs TCP probes against an allow-list of internal+external endpoints and a negative-list (cloud metadata etc.) every 10 minutes. Logs structured PASS/FAIL per check. Stack lives in admin/ansible under connectivity_tester/. One task per node via mode: global. Resolves Org1/hermes-voice-gateway#5 (connectivity testing blocker for #3). --- backend/configs/connectivity_tester.py | 62 ++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 backend/configs/connectivity_tester.py diff --git a/backend/configs/connectivity_tester.py b/backend/configs/connectivity_tester.py new file mode 100644 index 0000000..2937ae0 --- /dev/null +++ b/backend/configs/connectivity_tester.py @@ -0,0 +1,62 @@ +from enironment import Environment +from steps.checks import SimpleLog, UrlCheck +from steps.docker import DockerSwarmCheck, DockerSwarmDeploy +from steps.git import GitClone, CheckoutMerged, GitUnmerge +from steps.shared_state import SharedStateHolderInMemory + +clone = GitClone(remotes=[ + {"name": "gitea_hermes", "url": "https://gitea.rudolf.keenetic.link/hermes/ansible"}, + {"name": "gitea_admin", "url": "https://gitea.rudolf.keenetic.link/admin/ansible"}, +]) + +dockerSwarmCheck = DockerSwarmCheck( + stack_name="connectivity_tester", +) +unmerge = GitUnmerge(clone, dockerSwarmCheck) + +state = SharedStateHolderInMemory(unmerge=unmerge) + +checkoutMerged = CheckoutMerged(clone, + desired_branches=state, + push=False, + git_user_email="rudolfss13@gmail.com", + git_user_name="brencher_bot", + ) + +deployDocker = DockerSwarmDeploy( + wd=clone, + buildDocker=None, + stackChecker=dockerSwarmCheck, + envs=lambda: { + "version": "auto-" + checkoutMerged.progress().version, + }, + stack_name="connectivity_tester", + docker_compose_path="connectivity_tester/stack-compose.yml", +) + +checkPing = UrlCheck( + url="http://connectivity-tester.keenetic.link:8765/healthz", + expected={}, +) + +logUrls = SimpleLog(message={ + "userLinks": { + "Connectivity tester": "http://connectivity-tester.keenetic.link:8765/", + }, +}) + +__all__ = ["connectivity_tester"] +connectivity_tester = Environment( + id="connectivity_tester", + state=state, + pipeline=[ + clone, + state, + checkoutMerged, + dockerSwarmCheck, + unmerge, + deployDocker, + checkPing, + logUrls, + ], +) -- 2.54.0 From a3b53ce177d8bd4045104194a7cab9dac1951869 Mon Sep 17 00:00:00 2001 From: hermes Date: Mon, 14 Sep 2026 13:07:39 +0000 Subject: [PATCH 02/43] feat(configs): wire DockerComposeBuild into connectivity_tester pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit admin/ansible#39 has a build section in stack-compose.yml, so the image must be built locally — buildDocker=None would fail with 'No such image' on deploy. Added a proper DockerComposeBuild step between checkout and deploy. Image tagged auto- locally; no registry push (publish=False). --- backend/configs/connectivity_tester.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/backend/configs/connectivity_tester.py b/backend/configs/connectivity_tester.py index 2937ae0..7064bbf 100644 --- a/backend/configs/connectivity_tester.py +++ b/backend/configs/connectivity_tester.py @@ -1,6 +1,6 @@ from enironment import Environment from steps.checks import SimpleLog, UrlCheck -from steps.docker import DockerSwarmCheck, DockerSwarmDeploy +from steps.docker import DockerComposeBuild, DockerSwarmCheck, DockerSwarmDeploy from steps.git import GitClone, CheckoutMerged, GitUnmerge from steps.shared_state import SharedStateHolderInMemory @@ -23,9 +23,21 @@ checkoutMerged = CheckoutMerged(clone, git_user_name="brencher_bot", ) +buildDocker = DockerComposeBuild(clone, + docker_repo_username="", + docker_repo_password="", + docker_compose_path="connectivity_tester/stack-compose.yml", + docker_repo_url="https://registry.rudolf.keenetic.link", + publish=False, + build_cache=True, + envs=lambda: { + "version": "auto-" + checkoutMerged.progress().version, + }, + ) + deployDocker = DockerSwarmDeploy( wd=clone, - buildDocker=None, + buildDocker=buildDocker, stackChecker=dockerSwarmCheck, envs=lambda: { "version": "auto-" + checkoutMerged.progress().version, @@ -55,6 +67,7 @@ connectivity_tester = Environment( checkoutMerged, dockerSwarmCheck, unmerge, + buildDocker, deployDocker, checkPing, logUrls, -- 2.54.0 From be1e80a3b8228d0cbfe91fcb0bb970299d47bb7d Mon Sep 17 00:00:00 2001 From: hermes Date: Mon, 14 Sep 2026 17:03:41 +0000 Subject: [PATCH 03/43] feat(configs): relocate connectivity_tester stack into brencher repo Move compose + Python service + Dockerfile into backend/configs/connectivity_tester/ so the stack can be deployed from a single repo (no cross-repo compose lookup at deploy time). The brencher config now points at admin/brencher instead of admin/ansible, and at the new in-repo compose path. --- backend/configs/connectivity_tester.py | 8 +- .../configs/connectivity_tester/Dockerfile | 10 ++ backend/configs/connectivity_tester/app.py | 120 ++++++++++++++++++ .../configs/connectivity_tester/endpoints.py | 43 +++++++ .../connectivity_tester/stack-compose.yml | 44 +++++++ 5 files changed, 221 insertions(+), 4 deletions(-) create mode 100644 backend/configs/connectivity_tester/Dockerfile create mode 100644 backend/configs/connectivity_tester/app.py create mode 100644 backend/configs/connectivity_tester/endpoints.py create mode 100644 backend/configs/connectivity_tester/stack-compose.yml diff --git a/backend/configs/connectivity_tester.py b/backend/configs/connectivity_tester.py index 7064bbf..4f076ee 100644 --- a/backend/configs/connectivity_tester.py +++ b/backend/configs/connectivity_tester.py @@ -5,8 +5,8 @@ from steps.git import GitClone, CheckoutMerged, GitUnmerge from steps.shared_state import SharedStateHolderInMemory clone = GitClone(remotes=[ - {"name": "gitea_hermes", "url": "https://gitea.rudolf.keenetic.link/hermes/ansible"}, - {"name": "gitea_admin", "url": "https://gitea.rudolf.keenetic.link/admin/ansible"}, + {"name": "gitea_hermes", "url": "https://gitea.rudolf.keenetic.link/hermes/brencher"}, + {"name": "gitea_admin", "url": "https://gitea.rudolf.keenetic.link/admin/brencher"}, ]) dockerSwarmCheck = DockerSwarmCheck( @@ -26,7 +26,7 @@ checkoutMerged = CheckoutMerged(clone, buildDocker = DockerComposeBuild(clone, docker_repo_username="", docker_repo_password="", - docker_compose_path="connectivity_tester/stack-compose.yml", + docker_compose_path="backend/configs/connectivity_tester/stack-compose.yml", docker_repo_url="https://registry.rudolf.keenetic.link", publish=False, build_cache=True, @@ -43,7 +43,7 @@ deployDocker = DockerSwarmDeploy( "version": "auto-" + checkoutMerged.progress().version, }, stack_name="connectivity_tester", - docker_compose_path="connectivity_tester/stack-compose.yml", + docker_compose_path="backend/configs/connectivity_tester/stack-compose.yml", ) checkPing = UrlCheck( diff --git a/backend/configs/connectivity_tester/Dockerfile b/backend/configs/connectivity_tester/Dockerfile new file mode 100644 index 0000000..5d52445 --- /dev/null +++ b/backend/configs/connectivity_tester/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3.12-slim + +WORKDIR /app +COPY connectivity_tester/ /app/ + +ENV PYTHONUNBUFFERED=1 \ + PERIOD_SECONDS=600 \ + CHECK_TIMEOUT_S=3 + +CMD ["python", "-m", "app"] diff --git a/backend/configs/connectivity_tester/app.py b/backend/configs/connectivity_tester/app.py new file mode 100644 index 0000000..8795a83 --- /dev/null +++ b/backend/configs/connectivity_tester/app.py @@ -0,0 +1,120 @@ +"""Connectivity tester: verifies internal/external endpoints are reachable +and negative-list endpoints are not. Logs structured output, alerts on +unexpected failures. + +ENV: + EXTRA_ENDPOINTS - comma-separated host:port pairs, ad-hoc checks + PERIOD_SECONDS - sleep between full check rounds (default 600) + CHECK_TIMEOUT_S - per-endpoint timeout in seconds (default 3) +""" +from __future__ import annotations + +import asyncio +import logging +import os +import socket +import sys +import time +from typing import Iterable, List, Tuple + +from endpoints import EXPECTED_EXTERNAL, EXPECTED_INTERNAL, NEGATIVE, Endpoint + +LOG = logging.getLogger("connectivity-tester") + + +def _parse_extra(raw: str) -> List[Endpoint]: + out: List[Endpoint] = [] + for token in raw.split(","): + token = token.strip() + if not token: + continue + if ":" not in token: + continue + host, port_s = token.rsplit(":", 1) + try: + port = int(port_s) + except ValueError: + continue + out.append((host, port, "extra")) + return out + + +async def _tcp_check(host: str, port: int, timeout: float) -> Tuple[bool, str]: + try: + fut = asyncio.open_connection(host, port) + reader, writer = await asyncio.wait_for(fut, timeout=timeout) + writer.close() + try: + await writer.wait_closed() + except Exception: + pass + return True, "tcp ok" + except asyncio.TimeoutError: + return False, "timeout" + except OSError as exc: + return False, f"{type(exc).__name__}: {exc}" + + +async def _check_positive(endpoints: Iterable[Endpoint], timeout: float) -> int: + failures = 0 + for host, port, label in endpoints: + ok, detail = await _tcp_check(host, port, timeout) + if ok: + LOG.info("PASS %s:%d (%s) %s", host, port, label, detail) + else: + LOG.error("FAIL %s:%d (%s) %s", host, port, label, detail) + failures += 1 + return failures + + +async def _check_negative(endpoints: Iterable[Endpoint], timeout: float) -> int: + leaks = 0 + for host, port, label in endpoints: + ok, detail = await _tcp_check(host, port, timeout) + if ok: + LOG.error( + "LEAK %s:%d (%s) reachable but must NOT be: %s", + host, port, label, detail, + ) + leaks += 1 + else: + LOG.info("OK-NEG %s:%d (%s) blocked: %s", host, port, label, detail) + return leaks + + +async def _round(timeout: float) -> int: + extras = _parse_extra(os.environ.get("EXTRA_ENDPOINTS", "")) + LOG.info("--- round begin, %d internal + %d external + %d extra + %d negative ---", + len(EXPECTED_INTERNAL), len(EXPECTED_EXTERNAL), len(extras), len(NEGATIVE)) + failures = 0 + failures += await _check_positive(EXPECTED_INTERNAL, timeout) + failures += await _check_positive(EXPECTED_EXTERNAL, timeout) + failures += await _check_positive(extras, timeout) + leaks = await _check_negative(NEGATIVE, timeout) + failures += leaks + LOG.info("--- round end, %d failure(s) ---", failures) + return failures + + +async def main() -> int: + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(message)s", + stream=sys.stdout, + ) + period = int(os.environ.get("PERIOD_SECONDS", "600")) + timeout = float(os.environ.get("CHECK_TIMEOUT_S", "3")) + LOG.info("starting; period=%ds, check_timeout=%.1fs", period, timeout) + while True: + started = time.monotonic() + try: + await _round(timeout) + except Exception: + LOG.exception("round crashed") + elapsed = time.monotonic() - started + sleep_for = max(0.0, period - elapsed) + await asyncio.sleep(sleep_for) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/backend/configs/connectivity_tester/endpoints.py b/backend/configs/connectivity_tester/endpoints.py new file mode 100644 index 0000000..66e7300 --- /dev/null +++ b/backend/configs/connectivity_tester/endpoints.py @@ -0,0 +1,43 @@ +"""Endpoint allow-list and negative-list for connectivity-tester. + +EXPECTED_INTERNAL: TCP services in overlay networks that MUST be reachable. +EXPECTED_EXTERNAL: external hosts that MUST be reachable via squid egress. +NEGATIVE: hosts that MUST NOT respond (proxy / firewall sanity check). + +Each entry is (host, port, label). +""" +from __future__ import annotations + +from typing import List, Tuple + +Endpoint = Tuple[str, int, str] + + +EXPECTED_INTERNAL: List[Endpoint] = [ + ("squid", 3128, "egress proxy"), + ("hermes-gateway", 8000, "hermes messaging gateway"), + ("hermes-ui", 8087, "hermes dashboard ui"), + ("docker_socket_proxy", 2375, "read-only docker socket proxy"), + ("brencher-state-proxy", 80, "brencher state reverse proxy"), + ("gitea-server", 3000, "gitea http"), + ("act_runner", 8088, "gitea actions runner"), + ("telegram-mcp-server", 8000, "telegram mcp"), + ("gmail-mcp-server", 8000, "gmail mcp"), + ("voice-gateway", 8765, "voice realtime ws bridge"), + ("hermes-proxy", 8645, "openai-compatible http egress"), +] + + +EXPECTED_EXTERNAL: List[Endpoint] = [ + ("huggingface.co", 443, "piper voice model download"), + ("registry.rudolf.keenetic.link", 5000, "local docker registry"), + ("gitea.rudolf.keenetic.link", 443, "public gitea"), + ("pypi.org", 443, "python package index"), + ("github.com", 443, "github api"), +] + + +NEGATIVE: List[Endpoint] = [ + ("169.254.169.254", 80, "cloud metadata service"), + ("metadata.google.internal", 80, "gcp metadata"), +] diff --git a/backend/configs/connectivity_tester/stack-compose.yml b/backend/configs/connectivity_tester/stack-compose.yml new file mode 100644 index 0000000..486edde --- /dev/null +++ b/backend/configs/connectivity_tester/stack-compose.yml @@ -0,0 +1,44 @@ +name: connectivity_tester + +services: + connectivity-tester: + build: + context: . + dockerfile: Dockerfile + image: registry.rudolf.keenetic.link/connectivity_tester:${version} + hostname: connectivity-tester + user: "0:0" + networks: + - hermes_proxy + - default + environment: + PERIOD_SECONDS: "600" + CHECK_TIMEOUT_S: "3" + EXTRA_ENDPOINTS: "${EXTRA_ENDPOINTS:-}" + HTTP_PROXY: "http://squid:3128" + HTTPS_PROXY: "http://squid:3128" + NO_PROXY: "localhost,127.0.0.1,.local,squid,hermes-gateway,hermes-agent,hermes-proxy,brencher-state-proxy,docker_socket_proxy,10.0.0.0/8" + healthcheck: + test: ["CMD-SHELL", "python -c \"import socket; s=socket.socket(); s.settimeout(2); s.connect(('squid', 3128)); s.close()\""] + interval: 60s + timeout: 5s + retries: 3 + start_period: 30s + deploy: + mode: global + resources: + limits: + memory: 128M + pids: 64 + reservations: + memory: 64M + restart_policy: + condition: on-failure + delay: 10s + max_attempts: 3 + +networks: + hermes_proxy: + external: true + default: + external: true -- 2.54.0 From 3365a1496d4e9a9a9186e73d67c42ab635e0f32f Mon Sep 17 00:00:00 2001 From: hermes Date: Mon, 14 Sep 2026 17:26:56 +0000 Subject: [PATCH 04/43] chore(compose): add connectivity_tester to PROFILES env Without this, brencher's _discover_environments() does not see the connectivity_tester.py module on next restart. Adding to the comma- separated PROFILES env wires it in. Also undo the misplaced relocation from 4586136: clone remote and docker_compose_path point back at admin/ansible (the compose lives in admin/ansible#39, not in brencher). --- backend/configs/connectivity_tester.py | 8 +- .../configs/connectivity_tester/Dockerfile | 10 -- backend/configs/connectivity_tester/app.py | 120 ------------------ .../configs/connectivity_tester/endpoints.py | 43 ------- .../connectivity_tester/stack-compose.yml | 44 ------- docker-compose.yml | 2 +- 6 files changed, 5 insertions(+), 222 deletions(-) delete mode 100644 backend/configs/connectivity_tester/Dockerfile delete mode 100644 backend/configs/connectivity_tester/app.py delete mode 100644 backend/configs/connectivity_tester/endpoints.py delete mode 100644 backend/configs/connectivity_tester/stack-compose.yml diff --git a/backend/configs/connectivity_tester.py b/backend/configs/connectivity_tester.py index 4f076ee..7064bbf 100644 --- a/backend/configs/connectivity_tester.py +++ b/backend/configs/connectivity_tester.py @@ -5,8 +5,8 @@ from steps.git import GitClone, CheckoutMerged, GitUnmerge from steps.shared_state import SharedStateHolderInMemory clone = GitClone(remotes=[ - {"name": "gitea_hermes", "url": "https://gitea.rudolf.keenetic.link/hermes/brencher"}, - {"name": "gitea_admin", "url": "https://gitea.rudolf.keenetic.link/admin/brencher"}, + {"name": "gitea_hermes", "url": "https://gitea.rudolf.keenetic.link/hermes/ansible"}, + {"name": "gitea_admin", "url": "https://gitea.rudolf.keenetic.link/admin/ansible"}, ]) dockerSwarmCheck = DockerSwarmCheck( @@ -26,7 +26,7 @@ checkoutMerged = CheckoutMerged(clone, buildDocker = DockerComposeBuild(clone, docker_repo_username="", docker_repo_password="", - docker_compose_path="backend/configs/connectivity_tester/stack-compose.yml", + docker_compose_path="connectivity_tester/stack-compose.yml", docker_repo_url="https://registry.rudolf.keenetic.link", publish=False, build_cache=True, @@ -43,7 +43,7 @@ deployDocker = DockerSwarmDeploy( "version": "auto-" + checkoutMerged.progress().version, }, stack_name="connectivity_tester", - docker_compose_path="backend/configs/connectivity_tester/stack-compose.yml", + docker_compose_path="connectivity_tester/stack-compose.yml", ) checkPing = UrlCheck( diff --git a/backend/configs/connectivity_tester/Dockerfile b/backend/configs/connectivity_tester/Dockerfile deleted file mode 100644 index 5d52445..0000000 --- a/backend/configs/connectivity_tester/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -FROM python:3.12-slim - -WORKDIR /app -COPY connectivity_tester/ /app/ - -ENV PYTHONUNBUFFERED=1 \ - PERIOD_SECONDS=600 \ - CHECK_TIMEOUT_S=3 - -CMD ["python", "-m", "app"] diff --git a/backend/configs/connectivity_tester/app.py b/backend/configs/connectivity_tester/app.py deleted file mode 100644 index 8795a83..0000000 --- a/backend/configs/connectivity_tester/app.py +++ /dev/null @@ -1,120 +0,0 @@ -"""Connectivity tester: verifies internal/external endpoints are reachable -and negative-list endpoints are not. Logs structured output, alerts on -unexpected failures. - -ENV: - EXTRA_ENDPOINTS - comma-separated host:port pairs, ad-hoc checks - PERIOD_SECONDS - sleep between full check rounds (default 600) - CHECK_TIMEOUT_S - per-endpoint timeout in seconds (default 3) -""" -from __future__ import annotations - -import asyncio -import logging -import os -import socket -import sys -import time -from typing import Iterable, List, Tuple - -from endpoints import EXPECTED_EXTERNAL, EXPECTED_INTERNAL, NEGATIVE, Endpoint - -LOG = logging.getLogger("connectivity-tester") - - -def _parse_extra(raw: str) -> List[Endpoint]: - out: List[Endpoint] = [] - for token in raw.split(","): - token = token.strip() - if not token: - continue - if ":" not in token: - continue - host, port_s = token.rsplit(":", 1) - try: - port = int(port_s) - except ValueError: - continue - out.append((host, port, "extra")) - return out - - -async def _tcp_check(host: str, port: int, timeout: float) -> Tuple[bool, str]: - try: - fut = asyncio.open_connection(host, port) - reader, writer = await asyncio.wait_for(fut, timeout=timeout) - writer.close() - try: - await writer.wait_closed() - except Exception: - pass - return True, "tcp ok" - except asyncio.TimeoutError: - return False, "timeout" - except OSError as exc: - return False, f"{type(exc).__name__}: {exc}" - - -async def _check_positive(endpoints: Iterable[Endpoint], timeout: float) -> int: - failures = 0 - for host, port, label in endpoints: - ok, detail = await _tcp_check(host, port, timeout) - if ok: - LOG.info("PASS %s:%d (%s) %s", host, port, label, detail) - else: - LOG.error("FAIL %s:%d (%s) %s", host, port, label, detail) - failures += 1 - return failures - - -async def _check_negative(endpoints: Iterable[Endpoint], timeout: float) -> int: - leaks = 0 - for host, port, label in endpoints: - ok, detail = await _tcp_check(host, port, timeout) - if ok: - LOG.error( - "LEAK %s:%d (%s) reachable but must NOT be: %s", - host, port, label, detail, - ) - leaks += 1 - else: - LOG.info("OK-NEG %s:%d (%s) blocked: %s", host, port, label, detail) - return leaks - - -async def _round(timeout: float) -> int: - extras = _parse_extra(os.environ.get("EXTRA_ENDPOINTS", "")) - LOG.info("--- round begin, %d internal + %d external + %d extra + %d negative ---", - len(EXPECTED_INTERNAL), len(EXPECTED_EXTERNAL), len(extras), len(NEGATIVE)) - failures = 0 - failures += await _check_positive(EXPECTED_INTERNAL, timeout) - failures += await _check_positive(EXPECTED_EXTERNAL, timeout) - failures += await _check_positive(extras, timeout) - leaks = await _check_negative(NEGATIVE, timeout) - failures += leaks - LOG.info("--- round end, %d failure(s) ---", failures) - return failures - - -async def main() -> int: - logging.basicConfig( - level=logging.INFO, - format="%(asctime)s %(levelname)s %(message)s", - stream=sys.stdout, - ) - period = int(os.environ.get("PERIOD_SECONDS", "600")) - timeout = float(os.environ.get("CHECK_TIMEOUT_S", "3")) - LOG.info("starting; period=%ds, check_timeout=%.1fs", period, timeout) - while True: - started = time.monotonic() - try: - await _round(timeout) - except Exception: - LOG.exception("round crashed") - elapsed = time.monotonic() - started - sleep_for = max(0.0, period - elapsed) - await asyncio.sleep(sleep_for) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/backend/configs/connectivity_tester/endpoints.py b/backend/configs/connectivity_tester/endpoints.py deleted file mode 100644 index 66e7300..0000000 --- a/backend/configs/connectivity_tester/endpoints.py +++ /dev/null @@ -1,43 +0,0 @@ -"""Endpoint allow-list and negative-list for connectivity-tester. - -EXPECTED_INTERNAL: TCP services in overlay networks that MUST be reachable. -EXPECTED_EXTERNAL: external hosts that MUST be reachable via squid egress. -NEGATIVE: hosts that MUST NOT respond (proxy / firewall sanity check). - -Each entry is (host, port, label). -""" -from __future__ import annotations - -from typing import List, Tuple - -Endpoint = Tuple[str, int, str] - - -EXPECTED_INTERNAL: List[Endpoint] = [ - ("squid", 3128, "egress proxy"), - ("hermes-gateway", 8000, "hermes messaging gateway"), - ("hermes-ui", 8087, "hermes dashboard ui"), - ("docker_socket_proxy", 2375, "read-only docker socket proxy"), - ("brencher-state-proxy", 80, "brencher state reverse proxy"), - ("gitea-server", 3000, "gitea http"), - ("act_runner", 8088, "gitea actions runner"), - ("telegram-mcp-server", 8000, "telegram mcp"), - ("gmail-mcp-server", 8000, "gmail mcp"), - ("voice-gateway", 8765, "voice realtime ws bridge"), - ("hermes-proxy", 8645, "openai-compatible http egress"), -] - - -EXPECTED_EXTERNAL: List[Endpoint] = [ - ("huggingface.co", 443, "piper voice model download"), - ("registry.rudolf.keenetic.link", 5000, "local docker registry"), - ("gitea.rudolf.keenetic.link", 443, "public gitea"), - ("pypi.org", 443, "python package index"), - ("github.com", 443, "github api"), -] - - -NEGATIVE: List[Endpoint] = [ - ("169.254.169.254", 80, "cloud metadata service"), - ("metadata.google.internal", 80, "gcp metadata"), -] diff --git a/backend/configs/connectivity_tester/stack-compose.yml b/backend/configs/connectivity_tester/stack-compose.yml deleted file mode 100644 index 486edde..0000000 --- a/backend/configs/connectivity_tester/stack-compose.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: connectivity_tester - -services: - connectivity-tester: - build: - context: . - dockerfile: Dockerfile - image: registry.rudolf.keenetic.link/connectivity_tester:${version} - hostname: connectivity-tester - user: "0:0" - networks: - - hermes_proxy - - default - environment: - PERIOD_SECONDS: "600" - CHECK_TIMEOUT_S: "3" - EXTRA_ENDPOINTS: "${EXTRA_ENDPOINTS:-}" - HTTP_PROXY: "http://squid:3128" - HTTPS_PROXY: "http://squid:3128" - NO_PROXY: "localhost,127.0.0.1,.local,squid,hermes-gateway,hermes-agent,hermes-proxy,brencher-state-proxy,docker_socket_proxy,10.0.0.0/8" - healthcheck: - test: ["CMD-SHELL", "python -c \"import socket; s=socket.socket(); s.settimeout(2); s.connect(('squid', 3128)); s.close()\""] - interval: 60s - timeout: 5s - retries: 3 - start_period: 30s - deploy: - mode: global - resources: - limits: - memory: 128M - pids: 64 - reservations: - memory: 64M - restart_policy: - condition: on-failure - delay: 10s - max_attempts: 3 - -networks: - hermes_proxy: - external: true - default: - external: true diff --git a/docker-compose.yml b/docker-compose.yml index b4154d2..7d59b94 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,7 +17,7 @@ services: - VIRTUAL_HOST=brencher.rudolf.keenetic.link - LETSENCRYPT_HOST=brencher.rudolf.keenetic.link - VERSION=${version} - - PROFILES=brencher2,torrserv_proxy,immich,gmail_mcp,hermes,hermes_infra,gitea + - PROFILES=brencher2,torrserv_proxy,immich,gmail_mcp,hermes,hermes_infra,gitea,connectivity_tester - SECONDARY_BRENCHER=http://host.docker.internal:5002 deploy: replicas: 1 -- 2.54.0 From eccbfb806537df88c58036108229f9aef2044db3 Mon Sep 17 00:00:00 2001 From: hermes Date: Mon, 14 Sep 2026 17:45:09 +0000 Subject: [PATCH 05/43] fix(configs): use unmerge=None for connectivity_tester Without an entry in the brencher UI state.branches for connectivity_tester, GitUnmerge.progress() returns an empty set and the pipeline raises 'Expected exactly one version, got: set()'. This stalls the brencher loop, blocking other stacks (including brencher self-deploy) from being processed. Same workaround as hermes_infra.py (which uses unmerge=None because its stacks are managed via direct pushes, not via UI). --- backend/configs/connectivity_tester.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/configs/connectivity_tester.py b/backend/configs/connectivity_tester.py index 7064bbf..9d3e04f 100644 --- a/backend/configs/connectivity_tester.py +++ b/backend/configs/connectivity_tester.py @@ -12,7 +12,7 @@ clone = GitClone(remotes=[ dockerSwarmCheck = DockerSwarmCheck( stack_name="connectivity_tester", ) -unmerge = GitUnmerge(clone, dockerSwarmCheck) +unmerge = None state = SharedStateHolderInMemory(unmerge=unmerge) -- 2.54.0 From 65b2ef5325739e9247de26e442f18b908eac44d3 Mon Sep 17 00:00:00 2001 From: hermes Date: Mon, 14 Sep 2026 20:40:29 +0000 Subject: [PATCH 06/43] fix(configs): drop unmerge step from pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Same fix as hermes_infra.py — pass unmerge=None to SharedStateHolderInMemory and skip the unmerge step entirely. Having unmerge=None in the pipeline violates mypy's type signature (Pipeline expects AbstractStep, None is not assignable), causing the brencher image build to fail at the mypy step. This is the same workaround as hermes_infra (PR #83): the stack is managed via direct pushes, not via UI state, so unmerge is not needed. --- backend/configs/connectivity_tester.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/backend/configs/connectivity_tester.py b/backend/configs/connectivity_tester.py index 9d3e04f..c69b692 100644 --- a/backend/configs/connectivity_tester.py +++ b/backend/configs/connectivity_tester.py @@ -1,7 +1,7 @@ from enironment import Environment from steps.checks import SimpleLog, UrlCheck from steps.docker import DockerComposeBuild, DockerSwarmCheck, DockerSwarmDeploy -from steps.git import GitClone, CheckoutMerged, GitUnmerge +from steps.git import GitClone, CheckoutMerged from steps.shared_state import SharedStateHolderInMemory clone = GitClone(remotes=[ @@ -12,9 +12,8 @@ clone = GitClone(remotes=[ dockerSwarmCheck = DockerSwarmCheck( stack_name="connectivity_tester", ) -unmerge = None -state = SharedStateHolderInMemory(unmerge=unmerge) +state = SharedStateHolderInMemory(unmerge=None) checkoutMerged = CheckoutMerged(clone, desired_branches=state, @@ -66,7 +65,6 @@ connectivity_tester = Environment( state, checkoutMerged, dockerSwarmCheck, - unmerge, buildDocker, deployDocker, checkPing, -- 2.54.0 From 90555ca4ba0c4dbeb26a3de9af4cf816ef7addd1 Mon Sep 17 00:00:00 2001 From: brencher_bot Date: Tue, 15 Sep 2026 08:15:31 +0200 Subject: [PATCH 07/43] fix(configs/connectivity_tester): drop UrlCheck; DNS not configured yet Tracked in admin/brencher#68. --- backend/configs/connectivity_tester.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/backend/configs/connectivity_tester.py b/backend/configs/connectivity_tester.py index c69b692..f8afe27 100644 --- a/backend/configs/connectivity_tester.py +++ b/backend/configs/connectivity_tester.py @@ -1,5 +1,5 @@ from enironment import Environment -from steps.checks import SimpleLog, UrlCheck +from steps.checks import SimpleLog from steps.docker import DockerComposeBuild, DockerSwarmCheck, DockerSwarmDeploy from steps.git import GitClone, CheckoutMerged from steps.shared_state import SharedStateHolderInMemory @@ -45,11 +45,6 @@ deployDocker = DockerSwarmDeploy( docker_compose_path="connectivity_tester/stack-compose.yml", ) -checkPing = UrlCheck( - url="http://connectivity-tester.keenetic.link:8765/healthz", - expected={}, -) - logUrls = SimpleLog(message={ "userLinks": { "Connectivity tester": "http://connectivity-tester.keenetic.link:8765/", @@ -67,7 +62,6 @@ connectivity_tester = Environment( dockerSwarmCheck, buildDocker, deployDocker, - checkPing, logUrls, ], ) -- 2.54.0 From eabd348d923305a74efbc71ba1a2a421b8542e61 Mon Sep 17 00:00:00 2001 From: brencher_bot Date: Tue, 15 Sep 2026 13:34:31 +0200 Subject: [PATCH 08/43] fix(configs/connectivity_tester): wire unmerge for self-deploy Replace state=SharedStateHolderInMemory(unmerge=None) with a GitUnmerge(clone, dockerSwarmCheck)-backed state holder. Without this, desired_branches is always empty, and brencher's polling loop falls through to 'Empty branches set' on every cycle, so the stack only deploys when an operator PATCHes the environment through the UI. With the unmerge in place the swarm service's org.brencher.version label drives state.branches and the stack self-deploys on each polling cycle. --- backend/configs/connectivity_tester.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/configs/connectivity_tester.py b/backend/configs/connectivity_tester.py index f8afe27..abcd36d 100644 --- a/backend/configs/connectivity_tester.py +++ b/backend/configs/connectivity_tester.py @@ -1,7 +1,7 @@ from enironment import Environment from steps.checks import SimpleLog from steps.docker import DockerComposeBuild, DockerSwarmCheck, DockerSwarmDeploy -from steps.git import GitClone, CheckoutMerged +from steps.git import GitClone, CheckoutMerged, GitUnmerge from steps.shared_state import SharedStateHolderInMemory clone = GitClone(remotes=[ @@ -13,7 +13,8 @@ dockerSwarmCheck = DockerSwarmCheck( stack_name="connectivity_tester", ) -state = SharedStateHolderInMemory(unmerge=None) +unmerge = GitUnmerge(clone, dockerSwarmCheck) +state = SharedStateHolderInMemory(unmerge=unmerge) checkoutMerged = CheckoutMerged(clone, desired_branches=state, -- 2.54.0 From 0766ef6bdcfed476cdc48d3adc7a58f7b456c963 Mon Sep 17 00:00:00 2001 From: brencher_bot Date: Tue, 15 Sep 2026 13:58:24 +0200 Subject: [PATCH 09/43] docs(brencher): add AGENTS.md checklist for new stacks --- AGENTS.md | 99 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 AGENTS.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..e20e4e4 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,99 @@ +# Adding a new stack to brencher + +A "stack" is one self-contained Docker Swarm service that brencher manages +end-to-end: detect HEAD changes, build, deploy, and expose service status to +the UI. This file documents how to wire one up so it picks up commits +without an operator clicking "Update" in the UI every time. + +## Repository layout + +Brencher looks at exactly one commit on one ref per environment. For a new +stack `my_stack`: + +1. Push branch `feat/my_stack` (or whatever the team convention is) to a + Git repo listed in `remotes` of the config file. +2. Drop the repo in `gitea` or `github` so it's reachable through the same + remotes that other stacks use. +3. Make sure `my_stack/stack-compose.yml` and a Dockerfile (typically + `my_stack/Dockerfile`) exist at the path the config will point at. + +## Config file: `backend/configs/.py` + +The shape is fixed by `enironment.Environment` and the steps it pulls in. +Look at `backend/configs/hermes.py` for the canonical pattern. A new config +must: + +- Build a `GitClone` with at least one remote whose `url` points to the repo + above. +- Build a `DockerSwarmCheck(stack_name="")` to introspect the + deployed service. +- Build **`GitUnmerge(clone, dockerSwarmCheck)`** and wrap it into + `SharedStateHolderInMemory(unmerge=unmerge)`. This is what makes + autodeploy work — `GitUnmerge` reads the `org.brencher.version` label + from the running swarm task and converts it into + `desired_branches`, so each polling cycle asks "is HEAD ≠ what's + deployed?" without any UI input. +- Wire a `CheckoutMerged(clone, desired_branches=state, push=False, …)`. + Push stays `False` for first-party stacks; set `True` only when you want + brencher to push auto-named branches back to origin. +- Build a `DockerComposeBuild` (or `DockerSwarmBuild` if a Swarmfile is + used) pointing at the compose path in the repo. +- Build a `DockerSwarmDeploy(wd=clone, buildDocker=…, stackChecker=…, …)` + with the same `stack_name`. Image tag is `auto-` driven by + `checkoutMerged.progress().version`. +- Optionally append a `SimpleLog(message={"userLinks": {...}})` for the + link to surface in the UI. +- Set `__all__` and export the `Environment` named after the stack id. +- Add it to `Environment.pipeline` in the canonical order: + `clone → state → checkoutMerged → dockerSwarmCheck → build → deploy → + optionals`. Build and deploy are explicit `DockerComposeBuild` and + `DockerSwarmDeploy` siblings; do not collapse them into one step. + +If you skip the `GitUnmerge` line, `state.branches` will stay empty and +the stack will only deploy when an operator pushes a `branches` update +through the UI. Don't skip it; it's the difference between a self-driving +stack and a manual one. See `connectivity_tester.py` history for the bug +this caused. + +## Verification checklist + +- `python3 -c "from backend.configs.my_stack import my_stack"` works in + the brencher container. +- `PROFILES="my_stack" python3 backend/app.py` boots and the UI renders a + row for `my_stack` with non-empty branches (from `GitUnmerge`, + pointing at whatever image is currently deployed). +- Push to the configured branch, wait one full polling cycle + (~`RESET_INTERVAL` = 180s, plus the next pipeline tick), then confirm + the service was redeployed with a new `org.brencher.version` label. +- No UI `environment_update` was issued in between. + +## Common foot-guns + +- `dockerfile:` path in `stack-compose.yml` is interpreted relative to the + compose file's directory. The compose file and the Dockerfile must + live side-by-side; use `dockerfile: Dockerfile`, not + `dockerfile: `. +- Removing the `name:` key from the top of `stack-compose.yml` is fine + for `docker stack deploy` — the stack name comes from the CLI option + brencher sets. +- `EXTRA_ENDPOINTS` and friends are env vars; if you reference them in + compose, define them in the `envs=lambda: {...}` of the + `DockerComposeBuild` and `DockerSwarmDeploy`. `substitute` uses + `env.get(...)` for deploys but `env[...]` (raises) for builds, so an + unlisted variable kills the build but quietly produces an empty string + in the deploy. +- Compose files must not assume Docker networks that don't exist. Use + either an `external: true` net that already exists in Swarm, drop + the `networks:` block entirely, or `driver: overlay` for a + stack-private net. + +## Tracking + +When you add a stack, also: + +- Open a tracking issue in `admin/brencher` titled + `[ready-to-test] : …` with `state=` from your + `GitUnmerge`-derived branches and what URL/port the service exposes. +- Add the stack id to the `PROFILES` env var in the brencher stack + compose (`docker_stack` / Swarm declarative) so it gets picked up on + next rollout. The admin overlay is where this lives. -- 2.54.0 From f86aab5a1a21f1e158f98055aa7190c0f787a0c2 Mon Sep 17 00:00:00 2001 From: brencher_bot Date: Tue, 15 Sep 2026 20:25:42 +0200 Subject: [PATCH 10/43] chore(configs/connectivity_tester): try pushing to Gitea Container Registry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit publish=True + Gitea registry URL. push will fail with HTTP 401 until GITEA_REGISTRY_USERNAME/GITEA_REGISTRY_PASSWORD are wired in as brencher container env — that's the diagnostic to find out whether the bot token is actually allowed to push to gitea.rudolf.keenetic.link/hermes. --- backend/configs/connectivity_tester.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/backend/configs/connectivity_tester.py b/backend/configs/connectivity_tester.py index abcd36d..a61246a 100644 --- a/backend/configs/connectivity_tester.py +++ b/backend/configs/connectivity_tester.py @@ -1,3 +1,5 @@ +import os + from enironment import Environment from steps.checks import SimpleLog from steps.docker import DockerComposeBuild, DockerSwarmCheck, DockerSwarmDeploy @@ -23,12 +25,16 @@ checkoutMerged = CheckoutMerged(clone, git_user_name="brencher_bot", ) +GITEA_REGISTRY_USERNAME = os.environ.get("GITEA_REGISTRY_USERNAME", "") +GITEA_REGISTRY_PASSWORD = os.environ.get("GITEA_REGISTRY_PASSWORD", "") +GITEA_REGISTRY_URL = "https://gitea.rudolf.keenetic.link" + buildDocker = DockerComposeBuild(clone, - docker_repo_username="", - docker_repo_password="", + docker_repo_username=GITEA_REGISTRY_USERNAME, + docker_repo_password=GITEA_REGISTRY_PASSWORD, docker_compose_path="connectivity_tester/stack-compose.yml", - docker_repo_url="https://registry.rudolf.keenetic.link", - publish=False, + docker_repo_url=GITEA_REGISTRY_URL, + publish=True, build_cache=True, envs=lambda: { "version": "auto-" + checkoutMerged.progress().version, -- 2.54.0 From f7ff922606678501687b7d9192437eb6223fe50c Mon Sep 17 00:00:00 2001 From: brencher_bot Date: Thu, 17 Sep 2026 18:33:44 +0200 Subject: [PATCH 11/43] feat(configs): use PACKAGE_PUSH_USERNAME/TOKEN for Gitea CR; drop legacy registry --- backend/configs/connectivity_tester.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/configs/connectivity_tester.py b/backend/configs/connectivity_tester.py index a61246a..a666ca2 100644 --- a/backend/configs/connectivity_tester.py +++ b/backend/configs/connectivity_tester.py @@ -25,13 +25,13 @@ checkoutMerged = CheckoutMerged(clone, git_user_name="brencher_bot", ) -GITEA_REGISTRY_USERNAME = os.environ.get("GITEA_REGISTRY_USERNAME", "") -GITEA_REGISTRY_PASSWORD = os.environ.get("GITEA_REGISTRY_PASSWORD", "") +PACKAGE_PUSH_USERNAME = os.environ.get("PACKAGE_PUSH_USERNAME", "") +PACKAGE_PUSH_TOKEN = os.environ.get("PACKAGE_PUSH_TOKEN", "") GITEA_REGISTRY_URL = "https://gitea.rudolf.keenetic.link" buildDocker = DockerComposeBuild(clone, - docker_repo_username=GITEA_REGISTRY_USERNAME, - docker_repo_password=GITEA_REGISTRY_PASSWORD, + docker_repo_username=PACKAGE_PUSH_USERNAME, + docker_repo_password=PACKAGE_PUSH_TOKEN, docker_compose_path="connectivity_tester/stack-compose.yml", docker_repo_url=GITEA_REGISTRY_URL, publish=True, -- 2.54.0 From 65c855960d2f2bb23126d449d93d116fefc175e6 Mon Sep 17 00:00:00 2001 From: brencher_bot Date: Thu, 17 Sep 2026 19:27:02 +0200 Subject: [PATCH 12/43] feat(steps): GiteaActionsTrigger and GiteaActionsAwait for build dispatch --- backend/steps/actions.py | 153 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 backend/steps/actions.py diff --git a/backend/steps/actions.py b/backend/steps/actions.py new file mode 100644 index 0000000..fc0b7ca --- /dev/null +++ b/backend/steps/actions.py @@ -0,0 +1,153 @@ +import json +import logging +import os +import time +import urllib.error +import urllib.request +from dataclasses import dataclass +from typing import Any, Callable, Dict, Optional + +from enironment import AbstractStep +from steps.git import CheckoutMerged + +logger = logging.getLogger(__name__) + + +@dataclass +class GiteaActionsRun: + run_id: int + status: str + conclusion: Optional[str] + html_url: str + + +class GiteaActionsTrigger(AbstractStep[GiteaActionsRun]): + """Trigger a Gitea Actions workflow via the dispatches API.""" + + def __init__(self, + api_base: str, + owner: str, + repo: str, + workflow_file: str, + ref: str, + token: str, + inputs: Optional[Dict[str, str]] = None, + wd: Optional[CheckoutMerged] = None, + poll_timeout_sec: int = 1800, + poll_interval_sec: int = 10, + **kwargs: Any) -> None: + super().__init__(**kwargs) + self.api_base = api_base.rstrip("/") + self.owner = owner + self.repo = repo + self.workflow_file = workflow_file + self.ref = ref + self.token = token + self.inputs = inputs or {} + self.wd = wd + self.poll_timeout_sec = poll_timeout_sec + self.poll_interval_sec = poll_interval_sec + + def _http(self, method: str, url: str, body: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + data = None + headers = {"Authorization": f"token {self.token}", "Accept": "application/json"} + if body is not None: + data = json.dumps(body).encode() + headers["Content-Type"] = "application/json" + req = urllib.request.Request(url, data=data, headers=headers, method=method) + with urllib.request.urlopen(req, timeout=30) as r: + return json.loads(r.read().decode()) + + def _list_recent_runs(self) -> Optional[GiteaActionsRun]: + url = f"{self.api_base}/repos/{self.owner}/{self.repo}/actions/runs?limit=1" + try: + data = self._http("GET", url) + except urllib.error.HTTPError as e: + logger.error("list runs failed: %s", e) + return None + items = data.get("workflow_runs") or [] + if not items: + return None + r = items[0] + return GiteaActionsRun( + run_id=r["id"], + status=r.get("status", ""), + conclusion=r.get("conclusion"), + html_url=r.get("html_url", ""), + ) + + def progress(self) -> GiteaActionsRun: + latest = self._list_recent_runs() + body: Dict[str, Any] = {"ref": self.ref} + if self.inputs: + body["inputs"] = self.inputs + url = ( + f"{self.api_base}/repos/{self.owner}/{self.repo}" + f"/actions/workflows/{self.workflow_file}/dispatches" + ) + self._http("POST", url, body=body) + deadline = time.time() + self.poll_timeout_sec + while time.time() < deadline: + current = self._list_recent_runs() + if current is None or (latest is not None and current.run_id == latest.run_id): + time.sleep(self.poll_interval_sec) + continue + return current + raise TimeoutError( + f"Gitea Actions did not pick up a new run within {self.poll_timeout_sec}s" + ) + + +class GiteaActionsAwait(AbstractStep[GiteaActionsRun]): + """Wait for a previously-triggered Gitea Actions run to complete.""" + + def __init__(self, + api_base: str, + owner: str, + repo: str, + token: str, + run: Callable[[], GiteaActionsRun], + poll_timeout_sec: int = 1800, + poll_interval_sec: int = 10, + **kwargs: Any) -> None: + super().__init__(**kwargs) + self.api_base = api_base.rstrip("/") + self.owner = owner + self.repo = repo + self.token = token + self._run = run + self.poll_timeout_sec = poll_timeout_sec + self.poll_interval_sec = poll_interval_sec + + def _http(self, method: str, url: str) -> Dict[str, Any]: + headers = {"Authorization": f"token {self.token}", "Accept": "application/json"} + req = urllib.request.Request(url, headers=headers, method=method) + with urllib.request.urlopen(req, timeout=30) as r: + return json.loads(r.read().decode()) + + def progress(self) -> GiteaActionsRun: + run = self._run() + deadline = time.time() + self.poll_timeout_sec + while time.time() < deadline: + data = self._http( + "GET", + f"{self.api_base}/repos/{self.owner}/{self.repo}/actions/runs/{run.run_id}", + ) + status = data.get("status", "") + conclusion = data.get("conclusion") + if status == "completed": + if conclusion != "success": + raise RuntimeError( + f"Gitea Actions run {run.run_id} concluded with {conclusion}: " + f"{run.html_url}" + ) + return GiteaActionsRun( + run_id=run.run_id, + status=status, + conclusion=conclusion, + html_url=run.html_url, + ) + time.sleep(self.poll_interval_sec) + raise TimeoutError( + f"Gitea Actions run {run.run_id} did not complete within {self.poll_timeout_sec}s" + ) -- 2.54.0 From abdecd073630e43e8004e40d0d6c0abdfc61a62b Mon Sep 17 00:00:00 2001 From: brencher_bot Date: Thu, 17 Sep 2026 19:27:07 +0200 Subject: [PATCH 13/43] feat(steps): DockerComposePull for pulling prebuilt image --- backend/steps/docker_pull.py | 49 ++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 backend/steps/docker_pull.py diff --git a/backend/steps/docker_pull.py b/backend/steps/docker_pull.py new file mode 100644 index 0000000..93e5120 --- /dev/null +++ b/backend/steps/docker_pull.py @@ -0,0 +1,49 @@ +import json +import logging +import os +import re +import subprocess +from typing import Any, Callable, Dict + +import yaml +from enironment import AbstractStep +from steps.git import GitClone + +logger = logging.getLogger(__name__) + + +class DockerComposePull(AbstractStep[Dict[str, str]]): + """Pull images referenced by a docker-compose file from a remote registry.""" + + def __init__(self, + wd: GitClone, + docker_compose_path: str, + envs: Callable[[], Dict[str, Any]], + **kwargs: Any) -> None: + super().__init__(**kwargs) + self.wd = wd + self.envs = envs + self.docker_compose_path = docker_compose_path + + def progress(self) -> Dict[str, str]: + env = self.envs() + compose_path = os.path.join(self.wd.progress(), self.docker_compose_path) + with open(compose_path, "r") as f: + content = f.read() + content = re.sub(r"\$\{([^}]+)\}", lambda m: env.get(m.group(1), ""), content) + compose = yaml.safe_load(content) + for svc in compose.get("services", {}).values(): + if "build" in svc: + del svc["build"] + tmp_path = compose_path + ".pull.yml" + with open(tmp_path, "w") as f: + yaml.safe_dump(compose, f) + try: + subprocess.run( + ["docker", "compose", "-f", tmp_path, "pull"], + check=True, + cwd=self.wd.progress(), + ) + finally: + os.unlink(tmp_path) + return {name: svc.get("image", "") for name, svc in compose.get("services", {}).items()} -- 2.54.0 From 27413e4b9a2b62282de72cc407f5390e1a94b44b Mon Sep 17 00:00:00 2001 From: brencher_bot Date: Thu, 17 Sep 2026 19:27:12 +0200 Subject: [PATCH 14/43] feat(configs): brencher dispatches Actions workflow instead of build+push; deploy pulls image from Gitea CR --- backend/configs/connectivity_tester.py | 79 ++++++++++++++++---------- 1 file changed, 49 insertions(+), 30 deletions(-) diff --git a/backend/configs/connectivity_tester.py b/backend/configs/connectivity_tester.py index a666ca2..052bfe2 100644 --- a/backend/configs/connectivity_tester.py +++ b/backend/configs/connectivity_tester.py @@ -1,9 +1,11 @@ import os from enironment import Environment +from steps.actions import GiteaActionsAwait, GiteaActionsTrigger from steps.checks import SimpleLog -from steps.docker import DockerComposeBuild, DockerSwarmCheck, DockerSwarmDeploy -from steps.git import GitClone, CheckoutMerged, GitUnmerge +from steps.docker import DockerSwarmCheck, DockerSwarmDeploy +from steps.docker_pull import DockerComposePull +from steps.git import CheckoutMerged, GitClone, GitUnmerge from steps.shared_state import SharedStateHolderInMemory clone = GitClone(remotes=[ @@ -11,43 +13,58 @@ clone = GitClone(remotes=[ {"name": "gitea_admin", "url": "https://gitea.rudolf.keenetic.link/admin/ansible"}, ]) -dockerSwarmCheck = DockerSwarmCheck( - stack_name="connectivity_tester", -) +dockerSwarmCheck = DockerSwarmCheck(stack_name="connectivity_tester") unmerge = GitUnmerge(clone, dockerSwarmCheck) state = SharedStateHolderInMemory(unmerge=unmerge) -checkoutMerged = CheckoutMerged(clone, - desired_branches=state, - push=False, - git_user_email="rudolfss13@gmail.com", - git_user_name="brencher_bot", - ) +checkoutMerged = CheckoutMerged( + clone, + desired_branches=state, + push=False, + git_user_email="rudolfss13@gmail.com", + git_user_name="brencher_bot", +) -PACKAGE_PUSH_USERNAME = os.environ.get("PACKAGE_PUSH_USERNAME", "") -PACKAGE_PUSH_TOKEN = os.environ.get("PACKAGE_PUSH_TOKEN", "") -GITEA_REGISTRY_URL = "https://gitea.rudolf.keenetic.link" +GITEA_API = os.environ.get("GITEA_API", "https://gitea.rudolf.keenetic.link/api/v1") +GITEA_TOKEN = os.environ.get("GITEA_TOKEN", "") +WORKFLOW_REPO = os.environ.get("WORKFLOW_REPO", "hermes/ansible") +WORKFLOW_FILE = os.environ.get("WORKFLOW_FILE", "release.yml") +WORKFLOW_REF = os.environ.get("WORKFLOW_REF", "release/0.1.0-connectivity") -buildDocker = DockerComposeBuild(clone, - docker_repo_username=PACKAGE_PUSH_USERNAME, - docker_repo_password=PACKAGE_PUSH_TOKEN, - docker_compose_path="connectivity_tester/stack-compose.yml", - docker_repo_url=GITEA_REGISTRY_URL, - publish=True, - build_cache=True, - envs=lambda: { - "version": "auto-" + checkoutMerged.progress().version, - }, - ) +def _inputs() -> dict: + return {"tag": "auto-" + checkoutMerged.progress().version} + +trigger_build = GiteaActionsTrigger( + api_base=GITEA_API, + owner=WORKFLOW_REPO.split("/")[0], + repo=WORKFLOW_REPO.split("/")[1], + workflow_file=WORKFLOW_FILE, + ref=WORKFLOW_REF, + token=GITEA_TOKEN, + inputs=_inputs, + wd=checkoutMerged, +) + +await_build = GiteaActionsAwait( + api_base=GITEA_API, + owner=WORKFLOW_REPO.split("/")[0], + repo=WORKFLOW_REPO.split("/")[1], + token=GITEA_TOKEN, + run=lambda: trigger_build.progress(), +) + +pull_image = DockerComposePull( + clone, + docker_compose_path="connectivity_tester/stack-compose.yml", + envs=lambda: {"version": "auto-" + checkoutMerged.progress().version}, +) deployDocker = DockerSwarmDeploy( wd=clone, - buildDocker=buildDocker, + buildDocker=None, stackChecker=dockerSwarmCheck, - envs=lambda: { - "version": "auto-" + checkoutMerged.progress().version, - }, + envs=lambda: {"version": "auto-" + checkoutMerged.progress().version}, stack_name="connectivity_tester", docker_compose_path="connectivity_tester/stack-compose.yml", ) @@ -67,7 +84,9 @@ connectivity_tester = Environment( state, checkoutMerged, dockerSwarmCheck, - buildDocker, + trigger_build, + await_build, + pull_image, deployDocker, logUrls, ], -- 2.54.0 From 47057552436b0a23fa117081ef0306197b174164 Mon Sep 17 00:00:00 2001 From: brencher_bot Date: Thu, 17 Sep 2026 19:27:42 +0200 Subject: [PATCH 15/43] docs(AGENTS): document Actions build delegation pattern --- AGENTS.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index e20e4e4..52297b7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -97,3 +97,15 @@ When you add a stack, also: - Add the stack id to the `PROFILES` env var in the brencher stack compose (`docker_stack` / Swarm declarative) so it gets picked up on next rollout. The admin overlay is where this lives. + +## Build delegation pattern (Actions) + +Brencher does not run docker build or docker push itself. It dispatches a Gitea +Actions workflow via `POST /api/v1/repos/{owner}/{repo}/actions/workflows/{file}/dispatches`, +waits for the run to complete, then `docker compose pull` + `docker stack deploy`. + +Use `steps.actions.GiteaActionsTrigger` and `steps.actions.GiteaActionsAwait` +in the stack config, set `buildDocker=None` on `DockerSwarmDeploy`, and add +`DockerComposePull` before deploy. Credentials come from brencher env +(`GITEA_TOKEN`) and the compose image must point to a registry the runner can +push to (e.g. Gitea Container Registry `gitea.rudolf.keenetic.link//`). -- 2.54.0 From 07ef686273f8fa0117c2cc6c0a75d8c87e875784 Mon Sep 17 00:00:00 2001 From: hermes Date: Fri, 18 Sep 2026 13:10:39 +0000 Subject: [PATCH 16/43] fix(steps/actions): accept callable inputs (resolve at dispatch time) --- backend/steps/actions.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/backend/steps/actions.py b/backend/steps/actions.py index fc0b7ca..f06c54b 100644 --- a/backend/steps/actions.py +++ b/backend/steps/actions.py @@ -5,7 +5,7 @@ import time import urllib.error import urllib.request from dataclasses import dataclass -from typing import Any, Callable, Dict, Optional +from typing import Any, Callable, Dict, Optional, Union from enironment import AbstractStep from steps.git import CheckoutMerged @@ -21,6 +21,9 @@ class GiteaActionsRun: html_url: str +InputsSource = Union[Dict[str, str], Callable[[], Dict[str, str]], None] + + class GiteaActionsTrigger(AbstractStep[GiteaActionsRun]): """Trigger a Gitea Actions workflow via the dispatches API.""" @@ -31,7 +34,7 @@ class GiteaActionsTrigger(AbstractStep[GiteaActionsRun]): workflow_file: str, ref: str, token: str, - inputs: Optional[Dict[str, str]] = None, + inputs: InputsSource = None, wd: Optional[CheckoutMerged] = None, poll_timeout_sec: int = 1800, poll_interval_sec: int = 10, @@ -43,7 +46,7 @@ class GiteaActionsTrigger(AbstractStep[GiteaActionsRun]): self.workflow_file = workflow_file self.ref = ref self.token = token - self.inputs = inputs or {} + self.inputs = inputs self.wd = wd self.poll_timeout_sec = poll_timeout_sec self.poll_interval_sec = poll_interval_sec @@ -78,9 +81,14 @@ class GiteaActionsTrigger(AbstractStep[GiteaActionsRun]): def progress(self) -> GiteaActionsRun: latest = self._list_recent_runs() + resolved_inputs: Dict[str, str] = {} + if callable(self.inputs): + resolved_inputs = self.inputs() or {} + elif isinstance(self.inputs, dict): + resolved_inputs = self.inputs body: Dict[str, Any] = {"ref": self.ref} - if self.inputs: - body["inputs"] = self.inputs + if resolved_inputs: + body["inputs"] = resolved_inputs url = ( f"{self.api_base}/repos/{self.owner}/{self.repo}" f"/actions/workflows/{self.workflow_file}/dispatches" -- 2.54.0 From a1bac89a9604ddb743d1bc2bf4f55ab88b1739e0 Mon Sep 17 00:00:00 2001 From: hermes Date: Fri, 18 Sep 2026 20:42:53 +0000 Subject: [PATCH 17/43] fix(configs): WORKFLOW_REF=master (release/* doesn't trigger push) --- backend/configs/connectivity_tester.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/configs/connectivity_tester.py b/backend/configs/connectivity_tester.py index 052bfe2..93a41cf 100644 --- a/backend/configs/connectivity_tester.py +++ b/backend/configs/connectivity_tester.py @@ -30,7 +30,7 @@ GITEA_API = os.environ.get("GITEA_API", "https://gitea.rudolf.keenetic.link/api/ GITEA_TOKEN = os.environ.get("GITEA_TOKEN", "") WORKFLOW_REPO = os.environ.get("WORKFLOW_REPO", "hermes/ansible") WORKFLOW_FILE = os.environ.get("WORKFLOW_FILE", "release.yml") -WORKFLOW_REF = os.environ.get("WORKFLOW_REF", "release/0.1.0-connectivity") +WORKFLOW_REF = os.environ.get("WORKFLOW_REF", "master") def _inputs() -> dict: return {"tag": "auto-" + checkoutMerged.progress().version} -- 2.54.0 From c38622e1387683d2a0c41c888315c40120c61fef Mon Sep 17 00:00:00 2001 From: hermes Date: Fri, 18 Sep 2026 20:58:44 +0000 Subject: [PATCH 18/43] fix(steps/actions): clearer error when POST /dispatches 500 Refines the trigger_build failure mode: instead of opaque urllib.error.HTTPError, raises a RuntimeError explaining the gitea-side bug + the tag-push workaround (documented in the runtime message). --- backend/steps/actions.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/backend/steps/actions.py b/backend/steps/actions.py index f06c54b..9ca21da 100644 --- a/backend/steps/actions.py +++ b/backend/steps/actions.py @@ -93,7 +93,18 @@ class GiteaActionsTrigger(AbstractStep[GiteaActionsRun]): f"{self.api_base}/repos/{self.owner}/{self.repo}" f"/actions/workflows/{self.workflow_file}/dispatches" ) - self._http("POST", url, body=body) + try: + self._http("POST", url, body=body) + except urllib.error.HTTPError as e: + if e.code != 500: + raise + raise RuntimeError( + "POST /dispatches returned 500 (gitea-side bug on this instance). " + "Workaround: trigger the workflow manually by pushing a tag matching " + f"the workflow's `on.push.tags` filter, e.g. " + f"`git push HEAD:refs/tags/auto-`. " + f"Requested workflow: {self.owner}/{self.repo}/{self.workflow_file}@{self.ref}" + ) from e deadline = time.time() + self.poll_timeout_sec while time.time() < deadline: current = self._list_recent_runs() -- 2.54.0 From c1c1108b70b0fa711feb6b84d4ecf56604695475 Mon Sep 17 00:00:00 2001 From: hermes Date: Fri, 18 Sep 2026 21:39:31 +0000 Subject: [PATCH 19/43] fix(steps/git): use force-fetch refspec so force-pushed branches are picked up Default git fetch refspec refuses to update refs whose upstream SHA diverges from the local copy. When a branch is force-pushed (e.g. our smoke-test pushes of master, our re-applies of GITHUB_TOKEN fix), the local remote-tracking ref points at the old SHA and git refuses to overwrite it. This shows up as 'cannot lock ref ... is at but expected ' and fails the entire clone step. Adding '+refs/heads/*:refs/remotes//*' to each remote's refspec lets the fetch force-update the remote-tracking refs. Each remote's existing refspec is preserved; the force-refspec is appended only if not already present. --- backend/steps/git.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/backend/steps/git.py b/backend/steps/git.py index 8df31d2..fcdc3b5 100644 --- a/backend/steps/git.py +++ b/backend/steps/git.py @@ -103,13 +103,17 @@ class GitClone(AbstractStep[str]): remote.set_url(auth_url) else: remote = repo.create_remote(remote_name, auth_url) + force_refspec = f"+refs/heads/*:refs/remotes/{remote_name}/*" + existing_refspecs = list(remote.refspec) + if force_refspec not in existing_refspecs: + remote.set_refspec(remote.refspec + (force_refspec,)) if remote_name == "origin" and self.branchNamePrefix != "": with repo.config_writer() as config: config.set_value('remote "origin"', "fetch", f"+refs/heads/{self.branchNamePrefix}/*:refs/remotes/origin/{self.branchNamePrefix}/*") result = remote.fetch(prune=True) if not result or any(fetch_info.flags & fetch_info.ERROR for fetch_info in result): - raise BaseException(f"Failed to fetch updates for {repository_url}") + raise BaseException(f"Failed to fetch updates for {remote_name}") if not os.path.exists(os.path.join(self.repo_path, ".git")): raise BaseException(f"Failed to clone repositories {self.remotes} to {self.repo_path}") except BaseException as e: -- 2.54.0 From d3f09e40f38dd8de5b27ad7baf88b7d378c74885 Mon Sep 17 00:00:00 2001 From: hermes Date: Fri, 18 Sep 2026 21:48:09 +0000 Subject: [PATCH 20/43] Revert "fix(steps/git): use force-fetch refspec so force-pushed branches are picked up" This reverts commit 53b8796319ad5825591096d5b84d483ed0715194. --- backend/steps/git.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/backend/steps/git.py b/backend/steps/git.py index fcdc3b5..8df31d2 100644 --- a/backend/steps/git.py +++ b/backend/steps/git.py @@ -103,17 +103,13 @@ class GitClone(AbstractStep[str]): remote.set_url(auth_url) else: remote = repo.create_remote(remote_name, auth_url) - force_refspec = f"+refs/heads/*:refs/remotes/{remote_name}/*" - existing_refspecs = list(remote.refspec) - if force_refspec not in existing_refspecs: - remote.set_refspec(remote.refspec + (force_refspec,)) if remote_name == "origin" and self.branchNamePrefix != "": with repo.config_writer() as config: config.set_value('remote "origin"', "fetch", f"+refs/heads/{self.branchNamePrefix}/*:refs/remotes/origin/{self.branchNamePrefix}/*") result = remote.fetch(prune=True) if not result or any(fetch_info.flags & fetch_info.ERROR for fetch_info in result): - raise BaseException(f"Failed to fetch updates for {remote_name}") + raise BaseException(f"Failed to fetch updates for {repository_url}") if not os.path.exists(os.path.join(self.repo_path, ".git")): raise BaseException(f"Failed to clone repositories {self.remotes} to {self.repo_path}") except BaseException as e: -- 2.54.0 From fb5e9d14305c57267a9bfa6607a2de04f255664a Mon Sep 17 00:00:00 2001 From: hermes Date: Fri, 18 Sep 2026 21:48:34 +0000 Subject: [PATCH 21/43] fix(steps/git): use config_writer to set force-fetch refspec GitPython Remote does not expose a set_refspec method. Use repo.config_writer() to set remote..fetch '+refs/heads/*:refs/remotes//*'. This bypasses the default safety against updating remote-tracking refs that diverge from their upstream (force-pushed branches). Sets the refspec for every remote, not just 'origin', so multi-remote configs like connectivity_tester (hermes/ansible + admin/ansible) pick up force-pushed branches on either side. --- backend/steps/git.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/backend/steps/git.py b/backend/steps/git.py index 8df31d2..0556b8f 100644 --- a/backend/steps/git.py +++ b/backend/steps/git.py @@ -103,13 +103,18 @@ class GitClone(AbstractStep[str]): remote.set_url(auth_url) else: remote = repo.create_remote(remote_name, auth_url) + with repo.config_writer() as config: + config.set_value( + f'remote "{remote_name}".fetch', + f'+refs/heads/*:refs/remotes/{remote_name}/*', + ) if remote_name == "origin" and self.branchNamePrefix != "": with repo.config_writer() as config: config.set_value('remote "origin"', "fetch", f"+refs/heads/{self.branchNamePrefix}/*:refs/remotes/origin/{self.branchNamePrefix}/*") result = remote.fetch(prune=True) if not result or any(fetch_info.flags & fetch_info.ERROR for fetch_info in result): - raise BaseException(f"Failed to fetch updates for {repository_url}") + raise BaseException(f"Failed to fetch updates for {remote_name}") if not os.path.exists(os.path.join(self.repo_path, ".git")): raise BaseException(f"Failed to clone repositories {self.remotes} to {self.repo_path}") except BaseException as e: -- 2.54.0 From 10eb0a5d497485eb0c834470df892b9d8f2205a4 Mon Sep 17 00:00:00 2001 From: hermes Date: Fri, 18 Sep 2026 21:54:20 +0000 Subject: [PATCH 22/43] Revert "fix(steps/git): use config_writer to set force-fetch refspec" This reverts commit 3dd065773360c21387d6d9c5ca63b00067f938fc. --- backend/steps/git.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/backend/steps/git.py b/backend/steps/git.py index 0556b8f..8df31d2 100644 --- a/backend/steps/git.py +++ b/backend/steps/git.py @@ -103,18 +103,13 @@ class GitClone(AbstractStep[str]): remote.set_url(auth_url) else: remote = repo.create_remote(remote_name, auth_url) - with repo.config_writer() as config: - config.set_value( - f'remote "{remote_name}".fetch', - f'+refs/heads/*:refs/remotes/{remote_name}/*', - ) if remote_name == "origin" and self.branchNamePrefix != "": with repo.config_writer() as config: config.set_value('remote "origin"', "fetch", f"+refs/heads/{self.branchNamePrefix}/*:refs/remotes/origin/{self.branchNamePrefix}/*") result = remote.fetch(prune=True) if not result or any(fetch_info.flags & fetch_info.ERROR for fetch_info in result): - raise BaseException(f"Failed to fetch updates for {remote_name}") + raise BaseException(f"Failed to fetch updates for {repository_url}") if not os.path.exists(os.path.join(self.repo_path, ".git")): raise BaseException(f"Failed to clone repositories {self.remotes} to {self.repo_path}") except BaseException as e: -- 2.54.0 From 9b82c917bca2635dd314151d2b6e796874bae4e9 Mon Sep 17 00:00:00 2001 From: hermes Date: Fri, 18 Sep 2026 21:55:24 +0000 Subject: [PATCH 23/43] fix(steps/git): use config_writer with force-fetch refspec Default git fetch refspec refuses to update refs whose upstream SHA diverges from the local copy. When a branch is force-pushed (e.g. our smoke-test pushes of master, our re-applies of GITHUB_TOKEN fix), the local remote-tracking ref points at the old SHA and git refuses to overwrite it. This shows up as 'cannot lock ref ... is at but expected ' and fails the entire clone step. Adding '+refs/heads/*:refs/remotes//*' to each remote via config.set_value(section, option, value) lets the fetch force-update the remote-tracking refs. Each remote gets its own refspec; the existing 'origin' branchNamePrefix refspec is preserved. --- backend/steps/git.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/backend/steps/git.py b/backend/steps/git.py index 8df31d2..16db25d 100644 --- a/backend/steps/git.py +++ b/backend/steps/git.py @@ -103,13 +103,19 @@ class GitClone(AbstractStep[str]): remote.set_url(auth_url) else: remote = repo.create_remote(remote_name, auth_url) + with repo.config_writer() as config: + config.set_value( + f'remote "{remote_name}"', + 'fetch', + f'+refs/heads/*:refs/remotes/{remote_name}/*', + ) if remote_name == "origin" and self.branchNamePrefix != "": with repo.config_writer() as config: config.set_value('remote "origin"', "fetch", f"+refs/heads/{self.branchNamePrefix}/*:refs/remotes/origin/{self.branchNamePrefix}/*") result = remote.fetch(prune=True) if not result or any(fetch_info.flags & fetch_info.ERROR for fetch_info in result): - raise BaseException(f"Failed to fetch updates for {repository_url}") + raise BaseException(f"Failed to fetch updates for {remote_name}") if not os.path.exists(os.path.join(self.repo_path, ".git")): raise BaseException(f"Failed to clone repositories {self.remotes} to {self.repo_path}") except BaseException as e: -- 2.54.0 From 2b2f66afd09b23f76b4abceff87e0a69860c9459 Mon Sep 17 00:00:00 2001 From: hermes Date: Sat, 19 Sep 2026 08:32:55 +0000 Subject: [PATCH 24/43] docs(readme): add GITEA_TOKEN + Actions dispatch knobs to docker secret template The brencher stack build-delegation pattern (see AGENTS.md) needs four env vars to fire `GiteaActionsTrigger`: - GITEA_TOKEN: Gitea PAT used to POST /dispatches and poll /runs - WORKFLOW_REPO: owner/repo that hosts the workflow file (default: hermes/ansible) - WORKFLOW_FILE: workflow filename (default: release.yml) - WORKFLOW_REF: branch or tag the workflow is triggered on (default: master) Without GITEA_TOKEN the trigger step gets HTTP 401 from /api/v1/repos/{owner}/{repo}/actions/runs; the pipeline halts at trigger_build and the stack never deploys. Added the missing keys to the secret-create `printf` block in README.instruction.md so the next person wiring up a brencher stack doesn't have to discover this. Defaults shown match the connectivity_tester stack wiring (admin/brencher#65). --- README.instruction.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.instruction.md b/README.instruction.md index e48affa..4d506ad 100644 --- a/README.instruction.md +++ b/README.instruction.md @@ -33,7 +33,7 @@ docker service update --secret-rm brencher-secrets brencher_brencher-backend docker service update --secret-rm brencher-secrets brencher2_brencher-backend docker secret rm brencher-secrets -printf "GIT_USERNAME=git\nGIT_PASSWORD=TODO\n" | \ +printf "GIT_USERNAME=git\nGIT_PASSWORD=TODO\nGITEA_TOKEN=TODO\nWORKFLOW_REPO=hermes/ansible\nWORKFLOW_FILE=release.yml\nWORKFLOW_REF=master\n" | \ docker secret create brencher-secrets - docker service update --secret-add brencher-secrets brencher_brencher-backend -- 2.54.0 From f9eb7a1ffb7a49843f9200017493ebfc01a69ebd Mon Sep 17 00:00:00 2001 From: hermes Date: Sat, 19 Sep 2026 09:57:47 +0000 Subject: [PATCH 25/43] fix(configs): inline WORKFLOW_* in GiteaActions* ctor; keep only GITEA_TOKEN in secret The previous split between env vars and constructor args created two sources of truth for the workflow target. Moving WORKFLOW_OWNER / WORKFLOW_REPO / WORKFLOW_FILE / WORKFLOW_REF into the constructor arguments makes the dispatch target obvious at the call site and keeps the docker secret template minimal (only credentials it must contain). README.instruction.md now lists only GIT_USERNAME / GIT_PASSWORD / GITEA_TOKEN in the `printf` block; WORKFLOW_* are documented in AGENTS.md (already there) as stack-specific config knobs. --- README.instruction.md | 2 +- backend/configs/connectivity_tester.py | 16 +++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/README.instruction.md b/README.instruction.md index 4d506ad..883f4bc 100644 --- a/README.instruction.md +++ b/README.instruction.md @@ -33,7 +33,7 @@ docker service update --secret-rm brencher-secrets brencher_brencher-backend docker service update --secret-rm brencher-secrets brencher2_brencher-backend docker secret rm brencher-secrets -printf "GIT_USERNAME=git\nGIT_PASSWORD=TODO\nGITEA_TOKEN=TODO\nWORKFLOW_REPO=hermes/ansible\nWORKFLOW_FILE=release.yml\nWORKFLOW_REF=master\n" | \ +printf "GIT_USERNAME=git\nGIT_PASSWORD=TODO\nGITEA_TOKEN=TODO\n" | \ docker secret create brencher-secrets - docker service update --secret-add brencher-secrets brencher_brencher-backend diff --git a/backend/configs/connectivity_tester.py b/backend/configs/connectivity_tester.py index 93a41cf..e3b9677 100644 --- a/backend/configs/connectivity_tester.py +++ b/backend/configs/connectivity_tester.py @@ -28,17 +28,19 @@ checkoutMerged = CheckoutMerged( GITEA_API = os.environ.get("GITEA_API", "https://gitea.rudolf.keenetic.link/api/v1") GITEA_TOKEN = os.environ.get("GITEA_TOKEN", "") -WORKFLOW_REPO = os.environ.get("WORKFLOW_REPO", "hermes/ansible") -WORKFLOW_FILE = os.environ.get("WORKFLOW_FILE", "release.yml") -WORKFLOW_REF = os.environ.get("WORKFLOW_REF", "master") + +WORKFLOW_OWNER = "hermes" +WORKFLOW_REPO = "ansible" +WORKFLOW_FILE = "release.yml" +WORKFLOW_REF = "master" def _inputs() -> dict: return {"tag": "auto-" + checkoutMerged.progress().version} trigger_build = GiteaActionsTrigger( api_base=GITEA_API, - owner=WORKFLOW_REPO.split("/")[0], - repo=WORKFLOW_REPO.split("/")[1], + owner=WORKFLOW_OWNER, + repo=WORKFLOW_REPO, workflow_file=WORKFLOW_FILE, ref=WORKFLOW_REF, token=GITEA_TOKEN, @@ -48,8 +50,8 @@ trigger_build = GiteaActionsTrigger( await_build = GiteaActionsAwait( api_base=GITEA_API, - owner=WORKFLOW_REPO.split("/")[0], - repo=WORKFLOW_REPO.split("/")[1], + owner=WORKFLOW_OWNER, + repo=WORKFLOW_REPO, token=GITEA_TOKEN, run=lambda: trigger_build.progress(), ) -- 2.54.0 From 878ade00efc3eb95686b915807aea6d6a341a8ec Mon Sep 17 00:00:00 2001 From: hermes Date: Sat, 19 Sep 2026 11:38:22 +0000 Subject: [PATCH 26/43] fix(steps/actions): tolerate 204 No Content responses in _http POST /api/v1/repos/{owner}/{repo}/actions/workflows/{file}/dispatches returns HTTP 204 No Content on success (an empty body). The previous implementation unconditionally called json.loads on the response, which raised json.decoder.JSONDecodeError on the empty body. The stack hit this with 'Expecting value: line 1 column 1 (char 0)' and never advanced past GiteaActionsTrigger. Return {} instead for empty / 204 responses so the trigger step succeeds and the rest of the pipeline can run. --- backend/steps/actions.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/backend/steps/actions.py b/backend/steps/actions.py index 9ca21da..a4f2281 100644 --- a/backend/steps/actions.py +++ b/backend/steps/actions.py @@ -59,7 +59,10 @@ class GiteaActionsTrigger(AbstractStep[GiteaActionsRun]): headers["Content-Type"] = "application/json" req = urllib.request.Request(url, data=data, headers=headers, method=method) with urllib.request.urlopen(req, timeout=30) as r: - return json.loads(r.read().decode()) + body_bytes = r.read() + if not body_bytes or r.status == 204: + return {} + return json.loads(body_bytes.decode()) def _list_recent_runs(self) -> Optional[GiteaActionsRun]: url = f"{self.api_base}/repos/{self.owner}/{self.repo}/actions/runs?limit=1" -- 2.54.0 From d3a2a4e02a933230cd6b442f403e00ed7312b9d5 Mon Sep 17 00:00:00 2001 From: hermes Date: Sat, 19 Sep 2026 23:07:02 +0000 Subject: [PATCH 27/43] fix(configs): enable temporary branch push + dispatch on it Brencher now mirrors the auto-deploy-via-temp-branch pattern Alex described: instead of triggering the workflow on the default branch, brencher pushes the merge commit to `auto-` via CheckoutMerged's existing push=True logic, then dispatches the workflow on that new branch. The branch acts as a review point between build and deploy: after trigger_build and await_build complete, the deploy step still runs on the same local merge, but the workflow's provenance lives on a remote branch Alex can inspect or revert before merging back to default. - checkoutMerged.push = True (was False). CheckoutMerged already builds the branch name as `auto--...` from the sorted merges and force-pushes it to `origin` (see backend/steps/git.py ~line 311). - trigger_build ref pulls `checkoutMerged.progress().remote_branch_name` instead of a hard-coded "master". This makes the workflow run on the auto-branch brencher just pushed. --- backend/configs/connectivity_tester.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/backend/configs/connectivity_tester.py b/backend/configs/connectivity_tester.py index e3b9677..3eb51c7 100644 --- a/backend/configs/connectivity_tester.py +++ b/backend/configs/connectivity_tester.py @@ -21,7 +21,7 @@ state = SharedStateHolderInMemory(unmerge=unmerge) checkoutMerged = CheckoutMerged( clone, desired_branches=state, - push=False, + push=True, git_user_email="rudolfss13@gmail.com", git_user_name="brencher_bot", ) @@ -32,7 +32,6 @@ GITEA_TOKEN = os.environ.get("GITEA_TOKEN", "") WORKFLOW_OWNER = "hermes" WORKFLOW_REPO = "ansible" WORKFLOW_FILE = "release.yml" -WORKFLOW_REF = "master" def _inputs() -> dict: return {"tag": "auto-" + checkoutMerged.progress().version} @@ -42,7 +41,7 @@ trigger_build = GiteaActionsTrigger( owner=WORKFLOW_OWNER, repo=WORKFLOW_REPO, workflow_file=WORKFLOW_FILE, - ref=WORKFLOW_REF, + ref=checkoutMerged.progress().remote_branch_name or "master", token=GITEA_TOKEN, inputs=_inputs, wd=checkoutMerged, -- 2.54.0 From 0a53b649689c0592dc1df3247aaaaf642eb86653 Mon Sep 17 00:00:00 2001 From: hermes Date: Sun, 20 Sep 2026 08:58:49 +0000 Subject: [PATCH 28/43] Revert "fix(configs): enable temporary branch push + dispatch on it" This reverts commit 74229c167c6f42c1fd98f456d8ab764f1642ce90. --- backend/configs/connectivity_tester.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/configs/connectivity_tester.py b/backend/configs/connectivity_tester.py index 3eb51c7..e3b9677 100644 --- a/backend/configs/connectivity_tester.py +++ b/backend/configs/connectivity_tester.py @@ -21,7 +21,7 @@ state = SharedStateHolderInMemory(unmerge=unmerge) checkoutMerged = CheckoutMerged( clone, desired_branches=state, - push=True, + push=False, git_user_email="rudolfss13@gmail.com", git_user_name="brencher_bot", ) @@ -32,6 +32,7 @@ GITEA_TOKEN = os.environ.get("GITEA_TOKEN", "") WORKFLOW_OWNER = "hermes" WORKFLOW_REPO = "ansible" WORKFLOW_FILE = "release.yml" +WORKFLOW_REF = "master" def _inputs() -> dict: return {"tag": "auto-" + checkoutMerged.progress().version} @@ -41,7 +42,7 @@ trigger_build = GiteaActionsTrigger( owner=WORKFLOW_OWNER, repo=WORKFLOW_REPO, workflow_file=WORKFLOW_FILE, - ref=checkoutMerged.progress().remote_branch_name or "master", + ref=WORKFLOW_REF, token=GITEA_TOKEN, inputs=_inputs, wd=checkoutMerged, -- 2.54.0 From e34ab51ad45cbcd579aa419ac50ec96d5aa32bbf Mon Sep 17 00:00:00 2001 From: hermes Date: Sun, 20 Sep 2026 09:07:43 +0000 Subject: [PATCH 29/43] fix(steps/actions): accept Callable for ref to defer eval until progress() Previous attempt to dispatch on the auto-* branch failed at module load time: passing `ref=checkoutMerged.progress().remote_branch_name` runs the whole CheckoutMerged pipeline before brencher has populated self.env.id, raising BaseException('Environment not set for GitClone') and preventing brencher from even importing connectivity_tester. Add RefSource = Union[str, Callable[[], Optional[str]], None] mirroring the existing InputsSource pattern. The Callable is only invoked from within progress(), by which point the env is fully populated. The str form is preserved for existing callers (e.g. WORKFLOW_REF='master'). --- backend/configs/connectivity_tester.py | 5 ++--- backend/steps/actions.py | 8 +++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/backend/configs/connectivity_tester.py b/backend/configs/connectivity_tester.py index e3b9677..d02cefc 100644 --- a/backend/configs/connectivity_tester.py +++ b/backend/configs/connectivity_tester.py @@ -21,7 +21,7 @@ state = SharedStateHolderInMemory(unmerge=unmerge) checkoutMerged = CheckoutMerged( clone, desired_branches=state, - push=False, + push=True, git_user_email="rudolfss13@gmail.com", git_user_name="brencher_bot", ) @@ -32,7 +32,6 @@ GITEA_TOKEN = os.environ.get("GITEA_TOKEN", "") WORKFLOW_OWNER = "hermes" WORKFLOW_REPO = "ansible" WORKFLOW_FILE = "release.yml" -WORKFLOW_REF = "master" def _inputs() -> dict: return {"tag": "auto-" + checkoutMerged.progress().version} @@ -42,7 +41,7 @@ trigger_build = GiteaActionsTrigger( owner=WORKFLOW_OWNER, repo=WORKFLOW_REPO, workflow_file=WORKFLOW_FILE, - ref=WORKFLOW_REF, + ref=lambda: checkoutMerged.progress().remote_branch_name or "master", token=GITEA_TOKEN, inputs=_inputs, wd=checkoutMerged, diff --git a/backend/steps/actions.py b/backend/steps/actions.py index a4f2281..c805256 100644 --- a/backend/steps/actions.py +++ b/backend/steps/actions.py @@ -22,6 +22,7 @@ class GiteaActionsRun: InputsSource = Union[Dict[str, str], Callable[[], Dict[str, str]], None] +RefSource = Union[str, Callable[[], Optional[str]], None] class GiteaActionsTrigger(AbstractStep[GiteaActionsRun]): @@ -32,7 +33,7 @@ class GiteaActionsTrigger(AbstractStep[GiteaActionsRun]): owner: str, repo: str, workflow_file: str, - ref: str, + ref: RefSource, token: str, inputs: InputsSource = None, wd: Optional[CheckoutMerged] = None, @@ -89,7 +90,8 @@ class GiteaActionsTrigger(AbstractStep[GiteaActionsRun]): resolved_inputs = self.inputs() or {} elif isinstance(self.inputs, dict): resolved_inputs = self.inputs - body: Dict[str, Any] = {"ref": self.ref} + ref = self.ref() if callable(self.ref) else self.ref + body: Dict[str, Any] = {"ref": ref} if resolved_inputs: body["inputs"] = resolved_inputs url = ( @@ -106,7 +108,7 @@ class GiteaActionsTrigger(AbstractStep[GiteaActionsRun]): "Workaround: trigger the workflow manually by pushing a tag matching " f"the workflow's `on.push.tags` filter, e.g. " f"`git push HEAD:refs/tags/auto-`. " - f"Requested workflow: {self.owner}/{self.repo}/{self.workflow_file}@{self.ref}" + f"Requested workflow: {self.owner}/{self.repo}/{self.workflow_file}@{ref}" ) from e deadline = time.time() + self.poll_timeout_sec while time.time() < deadline: -- 2.54.0 From d9e87a7d2679abce1ac6a48ee441ff069f21cfd9 Mon Sep 17 00:00:00 2001 From: hermes Date: Sun, 20 Sep 2026 10:08:25 +0000 Subject: [PATCH 30/43] fix(steps/git): force-push auto-branch on every CheckoutMerged.progress() The 'remote_branch_name is None' guard made auto-branch push opt-in, not opt-out. When the merge commit already matched an existing remote-tracking ref (e.g. a feat/* branch the workflow was built on), brencher would skip the push and trigger the workflow on whatever branch happened to point at the merge commit. That branch is rarely the one the workflow is configured to watch, so the workflow would 404 or stay silent. Make auto-branch push unconditional when push=True: every cycle where brencher detects a new HEAD in its watched remotes gets a fresh 'auto-' branch on the origin remote, which the workflow trigger (on push to 'auto-*') picks up. The earlier 'remote_branch_name' lookup is kept so the returned value is useful for diagnostics, but no longer gates the push. --- backend/steps/git.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/steps/git.py b/backend/steps/git.py index 16db25d..0c56857 100644 --- a/backend/steps/git.py +++ b/backend/steps/git.py @@ -312,7 +312,7 @@ class CheckoutMerged(AbstractStep[CheckoutAndMergeResult]): remote_name, branch_name = ref.name.split('/', 1) remote_branch_name = branch_name if remote_name == 'origin' else ref.name break - if self.push and remote_branch_name is None: + if self.push: auto_branch_hash = hashlib.sha1(''.join([x.hexsha for x in sorted_commits]).encode()).hexdigest() auto_branch_name = f"auto-{version}" -- 2.54.0 From 41dc2f848510dbc29406ac6481a112494fb3b0a6 Mon Sep 17 00:00:00 2001 From: hermes Date: Sun, 20 Sep 2026 10:46:06 +0000 Subject: [PATCH 31/43] fix(configs/connectivity_tester): use GITEA_* env for git auth CheckoutMerged's auto-* push needs a token with write:repository on hermes/ansible (and admin/ansible). The brencher container reads creds via load_dotenv('/run/secrets/brencher-secrets') using GIT_USERNAME/GIT_PASSWORD by default. The PAT Alex provisioned (permissions: write:package, write:repository) is deployed into GITEA_TOKEN/GITEA_USERNAME, not GIT_TOKEN, so the push uses the wrong env vars and fails with 'Failed to authenticate user'. Passing credEnvPrefix='GITEA' to the GitClone for connectivity_tester makes brencher read GITEA_USERNAME + GITEA_PASSWORD for the auto push, leaving the existing GIT_* defaults intact for every other stack (so brencher.py / hermes_infra / brencher2 etc. keep working with whatever's currently in brencher-secrets). --- backend/configs/connectivity_tester.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/configs/connectivity_tester.py b/backend/configs/connectivity_tester.py index d02cefc..c1f7241 100644 --- a/backend/configs/connectivity_tester.py +++ b/backend/configs/connectivity_tester.py @@ -11,7 +11,9 @@ from steps.shared_state import SharedStateHolderInMemory clone = GitClone(remotes=[ {"name": "gitea_hermes", "url": "https://gitea.rudolf.keenetic.link/hermes/ansible"}, {"name": "gitea_admin", "url": "https://gitea.rudolf.keenetic.link/admin/ansible"}, -]) +], +credEnvPrefix="GITEA", +) dockerSwarmCheck = DockerSwarmCheck(stack_name="connectivity_tester") -- 2.54.0 From 01465fa1c595722382d572044b3cfdf7d3fc9cca Mon Sep 17 00:00:00 2001 From: hermes Date: Sun, 20 Sep 2026 11:34:30 +0000 Subject: [PATCH 32/43] fix(git): configure push remote for auto branches --- backend/configs/connectivity_tester.py | 2 +- backend/steps/git.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/backend/configs/connectivity_tester.py b/backend/configs/connectivity_tester.py index c1f7241..b81f981 100644 --- a/backend/configs/connectivity_tester.py +++ b/backend/configs/connectivity_tester.py @@ -23,7 +23,7 @@ state = SharedStateHolderInMemory(unmerge=unmerge) checkoutMerged = CheckoutMerged( clone, desired_branches=state, - push=True, + push="gitea_hermes", git_user_email="rudolfss13@gmail.com", git_user_name="brencher_bot", ) diff --git a/backend/steps/git.py b/backend/steps/git.py index 0c56857..93ca504 100644 --- a/backend/steps/git.py +++ b/backend/steps/git.py @@ -189,7 +189,7 @@ class CheckoutMerged(AbstractStep[CheckoutAndMergeResult]): desired_branches: AbstractStep[SharedState], git_user_email: str, git_user_name: str, - push: bool = True, + push: bool | str = True, **kwargs: Any): super().__init__(**kwargs) self.wd = wd @@ -315,9 +315,10 @@ class CheckoutMerged(AbstractStep[CheckoutAndMergeResult]): if self.push: auto_branch_hash = hashlib.sha1(''.join([x.hexsha for x in sorted_commits]).encode()).hexdigest() auto_branch_name = f"auto-{version}" + push_remote = self.push if isinstance(self.push, str) else "origin" logger.info(f"Pushing {repo.head.commit.hexsha} -> {auto_branch_name}") - repo.git.push('-f', 'origin', f"HEAD:refs/heads/{auto_branch_name}") + repo.git.push('-f', push_remote, f"HEAD:refs/heads/{auto_branch_name}") remote_branch_name = auto_branch_name return CheckoutAndMergeResult( -- 2.54.0 From 6479a3e604e4f9a8eaabdd2d339c75945538e5e5 Mon Sep 17 00:00:00 2001 From: hermes Date: Sun, 20 Sep 2026 11:37:17 +0000 Subject: [PATCH 33/43] fix(configs): push auto branch to admin remote --- backend/configs/connectivity_tester.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/configs/connectivity_tester.py b/backend/configs/connectivity_tester.py index b81f981..a10cba7 100644 --- a/backend/configs/connectivity_tester.py +++ b/backend/configs/connectivity_tester.py @@ -23,7 +23,7 @@ state = SharedStateHolderInMemory(unmerge=unmerge) checkoutMerged = CheckoutMerged( clone, desired_branches=state, - push="gitea_hermes", + push="gitea_admin", git_user_email="rudolfss13@gmail.com", git_user_name="brencher_bot", ) -- 2.54.0 From dd5ef3d27b55cc4104736d7123520faa1e64c85d Mon Sep 17 00:00:00 2001 From: hermes Date: Sun, 20 Sep 2026 12:01:01 +0000 Subject: [PATCH 34/43] fix(connectivity_tester): trigger admin workflow --- backend/configs/connectivity_tester.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/configs/connectivity_tester.py b/backend/configs/connectivity_tester.py index a10cba7..a421f04 100644 --- a/backend/configs/connectivity_tester.py +++ b/backend/configs/connectivity_tester.py @@ -31,7 +31,7 @@ checkoutMerged = CheckoutMerged( GITEA_API = os.environ.get("GITEA_API", "https://gitea.rudolf.keenetic.link/api/v1") GITEA_TOKEN = os.environ.get("GITEA_TOKEN", "") -WORKFLOW_OWNER = "hermes" +WORKFLOW_OWNER = "admin" WORKFLOW_REPO = "ansible" WORKFLOW_FILE = "release.yml" -- 2.54.0 From 575da88c05bccf97d0bd3afff2c5860d169da2d1 Mon Sep 17 00:00:00 2001 From: hermes Date: Sun, 20 Sep 2026 15:20:28 +0000 Subject: [PATCH 35/43] debug(web): log UI environment selections --- backend/web.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/backend/web.py b/backend/web.py index 5f8ae4c..313f6e9 100644 --- a/backend/web.py +++ b/backend/web.py @@ -94,6 +94,15 @@ class WebApp: merged = _merge_dicts(merged, connector.branches) return merged + def get_selection_summary(self) -> Dict[str, Dict[str, Any]]: + return { + env_id: { + "branches": env.get("branches", []), + "dry": env.get("dry"), + } + for env_id, env in self.get_global_envs_to_emit().items() + } + # --- Async scheduling from sync threads --- def _schedule_async(self, coro: Any) -> None: @@ -164,6 +173,7 @@ class WebApp: async def websocket_endpoint(self, websocket: WebSocket) -> None: await websocket.accept() state = self.ws_connections.setdefault(websocket, {}) + logger.info("UI connected: selected=%s", self.get_selection_summary()) try: # Send initial state on connect and record it so duplicate broadcasts are suppressed @@ -182,7 +192,12 @@ class WebApp: if "update" in message: update_data = message.get("update") or {} - logger.info(f"Received environment update: {update_data}") + logger.info( + "UI selection update: env=%s branches=%s dry=%s", + update_data.get('id', ''), + update_data.get('branches'), + update_data.get('dry'), + ) if self.secondaryManager: await self.secondaryManager.send({"update": update_data}) id = update_data.get('id', '') @@ -222,6 +237,7 @@ class WebApp: if isinstance(p, CachingStep): p.reset() logger.info(f"Updated environment {env.id} branches to {update_data.get('branches')}, dry={update_data.get('dry')}") + logger.info("UI selection resolved: selected=%s", self.get_selection_summary()) self.core.environment_update_event.set() await self.broadcast_environments(self.get_global_envs_to_emit()) -- 2.54.0 From 80d0a2c28270a4f655663ea2b75f98ad80f19ce0 Mon Sep 17 00:00:00 2001 From: hermes Date: Sun, 20 Sep 2026 18:04:26 +0000 Subject: [PATCH 36/43] fix(connectivity_tester): pass registry owner to compose --- backend/configs/connectivity_tester.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/backend/configs/connectivity_tester.py b/backend/configs/connectivity_tester.py index a421f04..193e0c4 100644 --- a/backend/configs/connectivity_tester.py +++ b/backend/configs/connectivity_tester.py @@ -60,14 +60,20 @@ await_build = GiteaActionsAwait( pull_image = DockerComposePull( clone, docker_compose_path="connectivity_tester/stack-compose.yml", - envs=lambda: {"version": "auto-" + checkoutMerged.progress().version}, + envs=lambda: { + "version": "auto-" + checkoutMerged.progress().version, + "registry_owner": "admin", + }, ) deployDocker = DockerSwarmDeploy( wd=clone, buildDocker=None, stackChecker=dockerSwarmCheck, - envs=lambda: {"version": "auto-" + checkoutMerged.progress().version}, + envs=lambda: { + "version": "auto-" + checkoutMerged.progress().version, + "registry_owner": "admin", + }, stack_name="connectivity_tester", docker_compose_path="connectivity_tester/stack-compose.yml", ) -- 2.54.0 From d34ed2d15da7eb17f0123c0f7c18c134fab65b68 Mon Sep 17 00:00:00 2001 From: hermes Date: Sun, 20 Sep 2026 18:46:22 +0000 Subject: [PATCH 37/43] fix(connectivity_tester): persist configured branch selection --- backend/configs/connectivity_tester.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/backend/configs/connectivity_tester.py b/backend/configs/connectivity_tester.py index 193e0c4..b880dd0 100644 --- a/backend/configs/connectivity_tester.py +++ b/backend/configs/connectivity_tester.py @@ -19,6 +19,9 @@ dockerSwarmCheck = DockerSwarmCheck(stack_name="connectivity_tester") unmerge = GitUnmerge(clone, dockerSwarmCheck) state = SharedStateHolderInMemory(unmerge=unmerge) +state.set_branches([ + ("gitea_hermes/fix/connectivity-tester-admin-registry", "HEAD"), +]) checkoutMerged = CheckoutMerged( clone, -- 2.54.0 From 9194e89197d9729d2f56d924d69a6d5aea914c90 Mon Sep 17 00:00:00 2001 From: hermes Date: Sun, 20 Sep 2026 19:03:57 +0000 Subject: [PATCH 38/43] fix(docker): login before private registry pull --- backend/steps/docker_pull.py | 37 ++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/backend/steps/docker_pull.py b/backend/steps/docker_pull.py index 93e5120..d9b23c1 100644 --- a/backend/steps/docker_pull.py +++ b/backend/steps/docker_pull.py @@ -4,6 +4,7 @@ import os import re import subprocess from typing import Any, Callable, Dict +from urllib.parse import urlparse import yaml from enironment import AbstractStep @@ -12,6 +13,40 @@ from steps.git import GitClone logger = logging.getLogger(__name__) +def _secret_values() -> Dict[str, str]: + values: Dict[str, str] = {} + try: + with open("/run/secrets/brencher-secrets") as f: + for line in f: + key, separator, value = line.strip().partition("=") + if separator: + values[key] = value + except FileNotFoundError: + pass + return values + + +def _registry_login(images: list[str]) -> None: + values = _secret_values() + username = values.get("REGISTRY_USERNAME") or values.get("GIT_USERNAME", "") + password = values.get("REGISTRY_TOKEN") or values.get("GIT_PASSWORD", "") + registries: set[str] = set() + for image in images: + registry = urlparse("https://" + image.split("/", 1)[0]).hostname + if registry: + registries.add(registry) + if not username or not password: + return + for registry in sorted(registries): + subprocess.run( + ["docker", "login", registry, "--username", username, "--password-stdin"], + input=password, + text=True, + check=True, + stdout=subprocess.DEVNULL, + ) + + class DockerComposePull(AbstractStep[Dict[str, str]]): """Pull images referenced by a docker-compose file from a remote registry.""" @@ -39,6 +74,8 @@ class DockerComposePull(AbstractStep[Dict[str, str]]): with open(tmp_path, "w") as f: yaml.safe_dump(compose, f) try: + images = [svc.get("image", "") for svc in compose.get("services", {}).values()] + _registry_login([image for image in images if image]) subprocess.run( ["docker", "compose", "-f", tmp_path, "pull"], check=True, -- 2.54.0 From 86aba667566d39d9087b66ac221085795085c778 Mon Sep 17 00:00:00 2001 From: hermes Date: Sun, 20 Sep 2026 19:12:42 +0000 Subject: [PATCH 39/43] fix(docker): reuse gitea credentials for registry pull --- backend/steps/docker_pull.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/backend/steps/docker_pull.py b/backend/steps/docker_pull.py index d9b23c1..deb7ef5 100644 --- a/backend/steps/docker_pull.py +++ b/backend/steps/docker_pull.py @@ -28,8 +28,16 @@ def _secret_values() -> Dict[str, str]: def _registry_login(images: list[str]) -> None: values = _secret_values() - username = values.get("REGISTRY_USERNAME") or values.get("GIT_USERNAME", "") - password = values.get("REGISTRY_TOKEN") or values.get("GIT_PASSWORD", "") + username = ( + values.get("REGISTRY_USERNAME") + or values.get("GITEA_USERNAME") + or values.get("GIT_USERNAME", "") + ) + password = ( + values.get("REGISTRY_TOKEN") + or values.get("GITEA_PASSWORD") + or values.get("GIT_PASSWORD", "") + ) registries: set[str] = set() for image in images: registry = urlparse("https://" + image.split("/", 1)[0]).hostname -- 2.54.0 From 7ba38ff72bfcb2fffbeebcab4d5449e7f430338a Mon Sep 17 00:00:00 2001 From: hermes Date: Sun, 20 Sep 2026 19:14:51 +0000 Subject: [PATCH 40/43] fix(docker): keep registry owner out of compose --- backend/steps/docker.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/steps/docker.py b/backend/steps/docker.py index 9c08d81..94712bc 100644 --- a/backend/steps/docker.py +++ b/backend/steps/docker.py @@ -230,6 +230,7 @@ class DockerSwarmDeploy(AbstractStep[str]): del svc["build"] svc["labels"] = {"org.brencher.version": env["version"]} del env["version"] + env.pop("registry_owner", None) merge_dicts(compose, env) # logger.info(f"Final compose: {compose}") content = yaml.safe_dump(compose) -- 2.54.0 From 0bdec9f1ec5db4bc6634a2c0e331c0e7ab285f22 Mon Sep 17 00:00:00 2001 From: hermes Date: Mon, 21 Sep 2026 06:41:19 +0000 Subject: [PATCH 41/43] fix(docker): use writable config for registry login --- backend/steps/docker_pull.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/backend/steps/docker_pull.py b/backend/steps/docker_pull.py index deb7ef5..3ffaefa 100644 --- a/backend/steps/docker_pull.py +++ b/backend/steps/docker_pull.py @@ -3,6 +3,7 @@ import logging import os import re import subprocess +import tempfile from typing import Any, Callable, Dict from urllib.parse import urlparse @@ -26,7 +27,7 @@ def _secret_values() -> Dict[str, str]: return values -def _registry_login(images: list[str]) -> None: +def _registry_login(images: list[str], env: Dict[str, str]) -> None: values = _secret_values() username = ( values.get("REGISTRY_USERNAME") @@ -51,6 +52,7 @@ def _registry_login(images: list[str]) -> None: input=password, text=True, check=True, + env=env, stdout=subprocess.DEVNULL, ) @@ -83,12 +85,16 @@ class DockerComposePull(AbstractStep[Dict[str, str]]): yaml.safe_dump(compose, f) try: images = [svc.get("image", "") for svc in compose.get("services", {}).values()] - _registry_login([image for image in images if image]) - subprocess.run( - ["docker", "compose", "-f", tmp_path, "pull"], - check=True, - cwd=self.wd.progress(), - ) + with tempfile.TemporaryDirectory() as docker_config: + env = os.environ.copy() + env["DOCKER_CONFIG"] = docker_config + _registry_login([image for image in images if image], env) + subprocess.run( + ["docker", "compose", "-f", tmp_path, "pull"], + check=True, + cwd=self.wd.progress(), + env=env, + ) finally: os.unlink(tmp_path) return {name: svc.get("image", "") for name, svc in compose.get("services", {}).items()} -- 2.54.0 From ebd23c63c432b78055b704aa594ff70672d4f947 Mon Sep 17 00:00:00 2001 From: hermes Date: Mon, 21 Sep 2026 17:15:57 +0000 Subject: [PATCH 42/43] fix(connectivity_tester): remove default branch selection --- backend/configs/connectivity_tester.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/backend/configs/connectivity_tester.py b/backend/configs/connectivity_tester.py index b880dd0..193e0c4 100644 --- a/backend/configs/connectivity_tester.py +++ b/backend/configs/connectivity_tester.py @@ -19,9 +19,6 @@ dockerSwarmCheck = DockerSwarmCheck(stack_name="connectivity_tester") unmerge = GitUnmerge(clone, dockerSwarmCheck) state = SharedStateHolderInMemory(unmerge=unmerge) -state.set_branches([ - ("gitea_hermes/fix/connectivity-tester-admin-registry", "HEAD"), -]) checkoutMerged = CheckoutMerged( clone, -- 2.54.0 From 390f1fe8ed91bbeba858a54092aa1c64464c539f Mon Sep 17 00:00:00 2001 From: hermes Date: Mon, 21 Sep 2026 17:17:03 +0000 Subject: [PATCH 43/43] refactor(connectivity_tester): deploy without compose pull --- AGENTS.md | 10 +-- backend/configs/connectivity_tester.py | 11 --- backend/steps/docker_pull.py | 100 ------------------------- 3 files changed, 5 insertions(+), 116 deletions(-) delete mode 100644 backend/steps/docker_pull.py diff --git a/AGENTS.md b/AGENTS.md index 52297b7..4338684 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -102,10 +102,10 @@ When you add a stack, also: Brencher does not run docker build or docker push itself. It dispatches a Gitea Actions workflow via `POST /api/v1/repos/{owner}/{repo}/actions/workflows/{file}/dispatches`, -waits for the run to complete, then `docker compose pull` + `docker stack deploy`. +waits for the run to complete, then runs `docker stack deploy`. Use `steps.actions.GiteaActionsTrigger` and `steps.actions.GiteaActionsAwait` -in the stack config, set `buildDocker=None` on `DockerSwarmDeploy`, and add -`DockerComposePull` before deploy. Credentials come from brencher env -(`GITEA_TOKEN`) and the compose image must point to a registry the runner can -push to (e.g. Gitea Container Registry `gitea.rudolf.keenetic.link//`). +in the stack config, set `buildDocker=None` on `DockerSwarmDeploy`, and let Swarm +resolve and pull the image on target nodes during deploy. Credentials must be +available to the deploy path and the compose image must point to a registry the +runner can push to (e.g. Gitea Container Registry `gitea.rudolf.keenetic.link//`). diff --git a/backend/configs/connectivity_tester.py b/backend/configs/connectivity_tester.py index 193e0c4..05d1426 100644 --- a/backend/configs/connectivity_tester.py +++ b/backend/configs/connectivity_tester.py @@ -4,7 +4,6 @@ from enironment import Environment from steps.actions import GiteaActionsAwait, GiteaActionsTrigger from steps.checks import SimpleLog from steps.docker import DockerSwarmCheck, DockerSwarmDeploy -from steps.docker_pull import DockerComposePull from steps.git import CheckoutMerged, GitClone, GitUnmerge from steps.shared_state import SharedStateHolderInMemory @@ -57,15 +56,6 @@ await_build = GiteaActionsAwait( run=lambda: trigger_build.progress(), ) -pull_image = DockerComposePull( - clone, - docker_compose_path="connectivity_tester/stack-compose.yml", - envs=lambda: { - "version": "auto-" + checkoutMerged.progress().version, - "registry_owner": "admin", - }, -) - deployDocker = DockerSwarmDeploy( wd=clone, buildDocker=None, @@ -95,7 +85,6 @@ connectivity_tester = Environment( dockerSwarmCheck, trigger_build, await_build, - pull_image, deployDocker, logUrls, ], diff --git a/backend/steps/docker_pull.py b/backend/steps/docker_pull.py deleted file mode 100644 index 3ffaefa..0000000 --- a/backend/steps/docker_pull.py +++ /dev/null @@ -1,100 +0,0 @@ -import json -import logging -import os -import re -import subprocess -import tempfile -from typing import Any, Callable, Dict -from urllib.parse import urlparse - -import yaml -from enironment import AbstractStep -from steps.git import GitClone - -logger = logging.getLogger(__name__) - - -def _secret_values() -> Dict[str, str]: - values: Dict[str, str] = {} - try: - with open("/run/secrets/brencher-secrets") as f: - for line in f: - key, separator, value = line.strip().partition("=") - if separator: - values[key] = value - except FileNotFoundError: - pass - return values - - -def _registry_login(images: list[str], env: Dict[str, str]) -> None: - values = _secret_values() - username = ( - values.get("REGISTRY_USERNAME") - or values.get("GITEA_USERNAME") - or values.get("GIT_USERNAME", "") - ) - password = ( - values.get("REGISTRY_TOKEN") - or values.get("GITEA_PASSWORD") - or values.get("GIT_PASSWORD", "") - ) - registries: set[str] = set() - for image in images: - registry = urlparse("https://" + image.split("/", 1)[0]).hostname - if registry: - registries.add(registry) - if not username or not password: - return - for registry in sorted(registries): - subprocess.run( - ["docker", "login", registry, "--username", username, "--password-stdin"], - input=password, - text=True, - check=True, - env=env, - stdout=subprocess.DEVNULL, - ) - - -class DockerComposePull(AbstractStep[Dict[str, str]]): - """Pull images referenced by a docker-compose file from a remote registry.""" - - def __init__(self, - wd: GitClone, - docker_compose_path: str, - envs: Callable[[], Dict[str, Any]], - **kwargs: Any) -> None: - super().__init__(**kwargs) - self.wd = wd - self.envs = envs - self.docker_compose_path = docker_compose_path - - def progress(self) -> Dict[str, str]: - env = self.envs() - compose_path = os.path.join(self.wd.progress(), self.docker_compose_path) - with open(compose_path, "r") as f: - content = f.read() - content = re.sub(r"\$\{([^}]+)\}", lambda m: env.get(m.group(1), ""), content) - compose = yaml.safe_load(content) - for svc in compose.get("services", {}).values(): - if "build" in svc: - del svc["build"] - tmp_path = compose_path + ".pull.yml" - with open(tmp_path, "w") as f: - yaml.safe_dump(compose, f) - try: - images = [svc.get("image", "") for svc in compose.get("services", {}).values()] - with tempfile.TemporaryDirectory() as docker_config: - env = os.environ.copy() - env["DOCKER_CONFIG"] = docker_config - _registry_login([image for image in images if image], env) - subprocess.run( - ["docker", "compose", "-f", tmp_path, "pull"], - check=True, - cwd=self.wd.progress(), - env=env, - ) - finally: - os.unlink(tmp_path) - return {name: svc.get("image", "") for name, svc in compose.get("services", {}).items()} -- 2.54.0