feat(configs): relocate connectivity_tester stack into brencher repo
Python package / build (3.12.3) (pull_request) Failing after 2m18s
Python package / build (3.13) (pull_request) Failing after 2m7s
Python package / build (3.14) (pull_request) Failing after 2m6s

Move compose + Python service + Dockerfile into backend/configs/connectivity_tester/
so the stack can be deployed from a single repo (no cross-repo compose
lookup at deploy time). The brencher config now points at
admin/brencher instead of admin/ansible, and at the new in-repo
compose path.
This commit is contained in:
hermes
2026-09-14 17:03:41 +00:00
parent 8b8bbbb230
commit 45861362c2
5 changed files with 221 additions and 4 deletions
+4 -4
View File
@@ -5,8 +5,8 @@ from steps.git import GitClone, CheckoutMerged, GitUnmerge
from steps.shared_state import SharedStateHolderInMemory
clone = GitClone(remotes=[
{"name": "gitea_hermes", "url": "https://gitea.rudolf.keenetic.link/hermes/ansible"},
{"name": "gitea_admin", "url": "https://gitea.rudolf.keenetic.link/admin/ansible"},
{"name": "gitea_hermes", "url": "https://gitea.rudolf.keenetic.link/hermes/brencher"},
{"name": "gitea_admin", "url": "https://gitea.rudolf.keenetic.link/admin/brencher"},
])
dockerSwarmCheck = DockerSwarmCheck(
@@ -26,7 +26,7 @@ checkoutMerged = CheckoutMerged(clone,
buildDocker = DockerComposeBuild(clone,
docker_repo_username="",
docker_repo_password="",
docker_compose_path="connectivity_tester/stack-compose.yml",
docker_compose_path="backend/configs/connectivity_tester/stack-compose.yml",
docker_repo_url="https://registry.rudolf.keenetic.link",
publish=False,
build_cache=True,
@@ -43,7 +43,7 @@ deployDocker = DockerSwarmDeploy(
"version": "auto-" + checkoutMerged.progress().version,
},
stack_name="connectivity_tester",
docker_compose_path="connectivity_tester/stack-compose.yml",
docker_compose_path="backend/configs/connectivity_tester/stack-compose.yml",
)
checkPing = UrlCheck(
@@ -0,0 +1,10 @@
FROM python:3.12-slim
WORKDIR /app
COPY connectivity_tester/ /app/
ENV PYTHONUNBUFFERED=1 \
PERIOD_SECONDS=600 \
CHECK_TIMEOUT_S=3
CMD ["python", "-m", "app"]
+120
View File
@@ -0,0 +1,120 @@
"""Connectivity tester: verifies internal/external endpoints are reachable
and negative-list endpoints are not. Logs structured output, alerts on
unexpected failures.
ENV:
EXTRA_ENDPOINTS - comma-separated host:port pairs, ad-hoc checks
PERIOD_SECONDS - sleep between full check rounds (default 600)
CHECK_TIMEOUT_S - per-endpoint timeout in seconds (default 3)
"""
from __future__ import annotations
import asyncio
import logging
import os
import socket
import sys
import time
from typing import Iterable, List, Tuple
from endpoints import EXPECTED_EXTERNAL, EXPECTED_INTERNAL, NEGATIVE, Endpoint
LOG = logging.getLogger("connectivity-tester")
def _parse_extra(raw: str) -> List[Endpoint]:
out: List[Endpoint] = []
for token in raw.split(","):
token = token.strip()
if not token:
continue
if ":" not in token:
continue
host, port_s = token.rsplit(":", 1)
try:
port = int(port_s)
except ValueError:
continue
out.append((host, port, "extra"))
return out
async def _tcp_check(host: str, port: int, timeout: float) -> Tuple[bool, str]:
try:
fut = asyncio.open_connection(host, port)
reader, writer = await asyncio.wait_for(fut, timeout=timeout)
writer.close()
try:
await writer.wait_closed()
except Exception:
pass
return True, "tcp ok"
except asyncio.TimeoutError:
return False, "timeout"
except OSError as exc:
return False, f"{type(exc).__name__}: {exc}"
async def _check_positive(endpoints: Iterable[Endpoint], timeout: float) -> int:
failures = 0
for host, port, label in endpoints:
ok, detail = await _tcp_check(host, port, timeout)
if ok:
LOG.info("PASS %s:%d (%s) %s", host, port, label, detail)
else:
LOG.error("FAIL %s:%d (%s) %s", host, port, label, detail)
failures += 1
return failures
async def _check_negative(endpoints: Iterable[Endpoint], timeout: float) -> int:
leaks = 0
for host, port, label in endpoints:
ok, detail = await _tcp_check(host, port, timeout)
if ok:
LOG.error(
"LEAK %s:%d (%s) reachable but must NOT be: %s",
host, port, label, detail,
)
leaks += 1
else:
LOG.info("OK-NEG %s:%d (%s) blocked: %s", host, port, label, detail)
return leaks
async def _round(timeout: float) -> int:
extras = _parse_extra(os.environ.get("EXTRA_ENDPOINTS", ""))
LOG.info("--- round begin, %d internal + %d external + %d extra + %d negative ---",
len(EXPECTED_INTERNAL), len(EXPECTED_EXTERNAL), len(extras), len(NEGATIVE))
failures = 0
failures += await _check_positive(EXPECTED_INTERNAL, timeout)
failures += await _check_positive(EXPECTED_EXTERNAL, timeout)
failures += await _check_positive(extras, timeout)
leaks = await _check_negative(NEGATIVE, timeout)
failures += leaks
LOG.info("--- round end, %d failure(s) ---", failures)
return failures
async def main() -> int:
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(levelname)s %(message)s",
stream=sys.stdout,
)
period = int(os.environ.get("PERIOD_SECONDS", "600"))
timeout = float(os.environ.get("CHECK_TIMEOUT_S", "3"))
LOG.info("starting; period=%ds, check_timeout=%.1fs", period, timeout)
while True:
started = time.monotonic()
try:
await _round(timeout)
except Exception:
LOG.exception("round crashed")
elapsed = time.monotonic() - started
sleep_for = max(0.0, period - elapsed)
await asyncio.sleep(sleep_for)
if __name__ == "__main__":
asyncio.run(main())
@@ -0,0 +1,43 @@
"""Endpoint allow-list and negative-list for connectivity-tester.
EXPECTED_INTERNAL: TCP services in overlay networks that MUST be reachable.
EXPECTED_EXTERNAL: external hosts that MUST be reachable via squid egress.
NEGATIVE: hosts that MUST NOT respond (proxy / firewall sanity check).
Each entry is (host, port, label).
"""
from __future__ import annotations
from typing import List, Tuple
Endpoint = Tuple[str, int, str]
EXPECTED_INTERNAL: List[Endpoint] = [
("squid", 3128, "egress proxy"),
("hermes-gateway", 8000, "hermes messaging gateway"),
("hermes-ui", 8087, "hermes dashboard ui"),
("docker_socket_proxy", 2375, "read-only docker socket proxy"),
("brencher-state-proxy", 80, "brencher state reverse proxy"),
("gitea-server", 3000, "gitea http"),
("act_runner", 8088, "gitea actions runner"),
("telegram-mcp-server", 8000, "telegram mcp"),
("gmail-mcp-server", 8000, "gmail mcp"),
("voice-gateway", 8765, "voice realtime ws bridge"),
("hermes-proxy", 8645, "openai-compatible http egress"),
]
EXPECTED_EXTERNAL: List[Endpoint] = [
("huggingface.co", 443, "piper voice model download"),
("registry.rudolf.keenetic.link", 5000, "local docker registry"),
("gitea.rudolf.keenetic.link", 443, "public gitea"),
("pypi.org", 443, "python package index"),
("github.com", 443, "github api"),
]
NEGATIVE: List[Endpoint] = [
("169.254.169.254", 80, "cloud metadata service"),
("metadata.google.internal", 80, "gcp metadata"),
]
@@ -0,0 +1,44 @@
name: connectivity_tester
services:
connectivity-tester:
build:
context: .
dockerfile: Dockerfile
image: registry.rudolf.keenetic.link/connectivity_tester:${version}
hostname: connectivity-tester
user: "0:0"
networks:
- hermes_proxy
- default
environment:
PERIOD_SECONDS: "600"
CHECK_TIMEOUT_S: "3"
EXTRA_ENDPOINTS: "${EXTRA_ENDPOINTS:-}"
HTTP_PROXY: "http://squid:3128"
HTTPS_PROXY: "http://squid:3128"
NO_PROXY: "localhost,127.0.0.1,.local,squid,hermes-gateway,hermes-agent,hermes-proxy,brencher-state-proxy,docker_socket_proxy,10.0.0.0/8"
healthcheck:
test: ["CMD-SHELL", "python -c \"import socket; s=socket.socket(); s.settimeout(2); s.connect(('squid', 3128)); s.close()\""]
interval: 60s
timeout: 5s
retries: 3
start_period: 30s
deploy:
mode: global
resources:
limits:
memory: 128M
pids: 64
reservations:
memory: 64M
restart_policy:
condition: on-failure
delay: 10s
max_attempts: 3
networks:
hermes_proxy:
external: true
default:
external: true