feat(configs): relocate connectivity_tester stack into brencher repo
Move compose + Python service + Dockerfile into backend/configs/connectivity_tester/ so the stack can be deployed from a single repo (no cross-repo compose lookup at deploy time). The brencher config now points at admin/brencher instead of admin/ansible, and at the new in-repo compose path.
This commit is contained in:
@@ -5,8 +5,8 @@ from steps.git import GitClone, CheckoutMerged, GitUnmerge
|
||||
from steps.shared_state import SharedStateHolderInMemory
|
||||
|
||||
clone = GitClone(remotes=[
|
||||
{"name": "gitea_hermes", "url": "https://gitea.rudolf.keenetic.link/hermes/ansible"},
|
||||
{"name": "gitea_admin", "url": "https://gitea.rudolf.keenetic.link/admin/ansible"},
|
||||
{"name": "gitea_hermes", "url": "https://gitea.rudolf.keenetic.link/hermes/brencher"},
|
||||
{"name": "gitea_admin", "url": "https://gitea.rudolf.keenetic.link/admin/brencher"},
|
||||
])
|
||||
|
||||
dockerSwarmCheck = DockerSwarmCheck(
|
||||
@@ -26,7 +26,7 @@ checkoutMerged = CheckoutMerged(clone,
|
||||
buildDocker = DockerComposeBuild(clone,
|
||||
docker_repo_username="",
|
||||
docker_repo_password="",
|
||||
docker_compose_path="connectivity_tester/stack-compose.yml",
|
||||
docker_compose_path="backend/configs/connectivity_tester/stack-compose.yml",
|
||||
docker_repo_url="https://registry.rudolf.keenetic.link",
|
||||
publish=False,
|
||||
build_cache=True,
|
||||
@@ -43,7 +43,7 @@ deployDocker = DockerSwarmDeploy(
|
||||
"version": "auto-" + checkoutMerged.progress().version,
|
||||
},
|
||||
stack_name="connectivity_tester",
|
||||
docker_compose_path="connectivity_tester/stack-compose.yml",
|
||||
docker_compose_path="backend/configs/connectivity_tester/stack-compose.yml",
|
||||
)
|
||||
|
||||
checkPing = UrlCheck(
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
FROM python:3.12-slim
|
||||
|
||||
WORKDIR /app
|
||||
COPY connectivity_tester/ /app/
|
||||
|
||||
ENV PYTHONUNBUFFERED=1 \
|
||||
PERIOD_SECONDS=600 \
|
||||
CHECK_TIMEOUT_S=3
|
||||
|
||||
CMD ["python", "-m", "app"]
|
||||
@@ -0,0 +1,120 @@
|
||||
"""Connectivity tester: verifies internal/external endpoints are reachable
|
||||
and negative-list endpoints are not. Logs structured output, alerts on
|
||||
unexpected failures.
|
||||
|
||||
ENV:
|
||||
EXTRA_ENDPOINTS - comma-separated host:port pairs, ad-hoc checks
|
||||
PERIOD_SECONDS - sleep between full check rounds (default 600)
|
||||
CHECK_TIMEOUT_S - per-endpoint timeout in seconds (default 3)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import socket
|
||||
import sys
|
||||
import time
|
||||
from typing import Iterable, List, Tuple
|
||||
|
||||
from endpoints import EXPECTED_EXTERNAL, EXPECTED_INTERNAL, NEGATIVE, Endpoint
|
||||
|
||||
LOG = logging.getLogger("connectivity-tester")
|
||||
|
||||
|
||||
def _parse_extra(raw: str) -> List[Endpoint]:
|
||||
out: List[Endpoint] = []
|
||||
for token in raw.split(","):
|
||||
token = token.strip()
|
||||
if not token:
|
||||
continue
|
||||
if ":" not in token:
|
||||
continue
|
||||
host, port_s = token.rsplit(":", 1)
|
||||
try:
|
||||
port = int(port_s)
|
||||
except ValueError:
|
||||
continue
|
||||
out.append((host, port, "extra"))
|
||||
return out
|
||||
|
||||
|
||||
async def _tcp_check(host: str, port: int, timeout: float) -> Tuple[bool, str]:
|
||||
try:
|
||||
fut = asyncio.open_connection(host, port)
|
||||
reader, writer = await asyncio.wait_for(fut, timeout=timeout)
|
||||
writer.close()
|
||||
try:
|
||||
await writer.wait_closed()
|
||||
except Exception:
|
||||
pass
|
||||
return True, "tcp ok"
|
||||
except asyncio.TimeoutError:
|
||||
return False, "timeout"
|
||||
except OSError as exc:
|
||||
return False, f"{type(exc).__name__}: {exc}"
|
||||
|
||||
|
||||
async def _check_positive(endpoints: Iterable[Endpoint], timeout: float) -> int:
|
||||
failures = 0
|
||||
for host, port, label in endpoints:
|
||||
ok, detail = await _tcp_check(host, port, timeout)
|
||||
if ok:
|
||||
LOG.info("PASS %s:%d (%s) %s", host, port, label, detail)
|
||||
else:
|
||||
LOG.error("FAIL %s:%d (%s) %s", host, port, label, detail)
|
||||
failures += 1
|
||||
return failures
|
||||
|
||||
|
||||
async def _check_negative(endpoints: Iterable[Endpoint], timeout: float) -> int:
|
||||
leaks = 0
|
||||
for host, port, label in endpoints:
|
||||
ok, detail = await _tcp_check(host, port, timeout)
|
||||
if ok:
|
||||
LOG.error(
|
||||
"LEAK %s:%d (%s) reachable but must NOT be: %s",
|
||||
host, port, label, detail,
|
||||
)
|
||||
leaks += 1
|
||||
else:
|
||||
LOG.info("OK-NEG %s:%d (%s) blocked: %s", host, port, label, detail)
|
||||
return leaks
|
||||
|
||||
|
||||
async def _round(timeout: float) -> int:
|
||||
extras = _parse_extra(os.environ.get("EXTRA_ENDPOINTS", ""))
|
||||
LOG.info("--- round begin, %d internal + %d external + %d extra + %d negative ---",
|
||||
len(EXPECTED_INTERNAL), len(EXPECTED_EXTERNAL), len(extras), len(NEGATIVE))
|
||||
failures = 0
|
||||
failures += await _check_positive(EXPECTED_INTERNAL, timeout)
|
||||
failures += await _check_positive(EXPECTED_EXTERNAL, timeout)
|
||||
failures += await _check_positive(extras, timeout)
|
||||
leaks = await _check_negative(NEGATIVE, timeout)
|
||||
failures += leaks
|
||||
LOG.info("--- round end, %d failure(s) ---", failures)
|
||||
return failures
|
||||
|
||||
|
||||
async def main() -> int:
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s %(levelname)s %(message)s",
|
||||
stream=sys.stdout,
|
||||
)
|
||||
period = int(os.environ.get("PERIOD_SECONDS", "600"))
|
||||
timeout = float(os.environ.get("CHECK_TIMEOUT_S", "3"))
|
||||
LOG.info("starting; period=%ds, check_timeout=%.1fs", period, timeout)
|
||||
while True:
|
||||
started = time.monotonic()
|
||||
try:
|
||||
await _round(timeout)
|
||||
except Exception:
|
||||
LOG.exception("round crashed")
|
||||
elapsed = time.monotonic() - started
|
||||
sleep_for = max(0.0, period - elapsed)
|
||||
await asyncio.sleep(sleep_for)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,43 @@
|
||||
"""Endpoint allow-list and negative-list for connectivity-tester.
|
||||
|
||||
EXPECTED_INTERNAL: TCP services in overlay networks that MUST be reachable.
|
||||
EXPECTED_EXTERNAL: external hosts that MUST be reachable via squid egress.
|
||||
NEGATIVE: hosts that MUST NOT respond (proxy / firewall sanity check).
|
||||
|
||||
Each entry is (host, port, label).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import List, Tuple
|
||||
|
||||
Endpoint = Tuple[str, int, str]
|
||||
|
||||
|
||||
EXPECTED_INTERNAL: List[Endpoint] = [
|
||||
("squid", 3128, "egress proxy"),
|
||||
("hermes-gateway", 8000, "hermes messaging gateway"),
|
||||
("hermes-ui", 8087, "hermes dashboard ui"),
|
||||
("docker_socket_proxy", 2375, "read-only docker socket proxy"),
|
||||
("brencher-state-proxy", 80, "brencher state reverse proxy"),
|
||||
("gitea-server", 3000, "gitea http"),
|
||||
("act_runner", 8088, "gitea actions runner"),
|
||||
("telegram-mcp-server", 8000, "telegram mcp"),
|
||||
("gmail-mcp-server", 8000, "gmail mcp"),
|
||||
("voice-gateway", 8765, "voice realtime ws bridge"),
|
||||
("hermes-proxy", 8645, "openai-compatible http egress"),
|
||||
]
|
||||
|
||||
|
||||
EXPECTED_EXTERNAL: List[Endpoint] = [
|
||||
("huggingface.co", 443, "piper voice model download"),
|
||||
("registry.rudolf.keenetic.link", 5000, "local docker registry"),
|
||||
("gitea.rudolf.keenetic.link", 443, "public gitea"),
|
||||
("pypi.org", 443, "python package index"),
|
||||
("github.com", 443, "github api"),
|
||||
]
|
||||
|
||||
|
||||
NEGATIVE: List[Endpoint] = [
|
||||
("169.254.169.254", 80, "cloud metadata service"),
|
||||
("metadata.google.internal", 80, "gcp metadata"),
|
||||
]
|
||||
@@ -0,0 +1,44 @@
|
||||
name: connectivity_tester
|
||||
|
||||
services:
|
||||
connectivity-tester:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
image: registry.rudolf.keenetic.link/connectivity_tester:${version}
|
||||
hostname: connectivity-tester
|
||||
user: "0:0"
|
||||
networks:
|
||||
- hermes_proxy
|
||||
- default
|
||||
environment:
|
||||
PERIOD_SECONDS: "600"
|
||||
CHECK_TIMEOUT_S: "3"
|
||||
EXTRA_ENDPOINTS: "${EXTRA_ENDPOINTS:-}"
|
||||
HTTP_PROXY: "http://squid:3128"
|
||||
HTTPS_PROXY: "http://squid:3128"
|
||||
NO_PROXY: "localhost,127.0.0.1,.local,squid,hermes-gateway,hermes-agent,hermes-proxy,brencher-state-proxy,docker_socket_proxy,10.0.0.0/8"
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "python -c \"import socket; s=socket.socket(); s.settimeout(2); s.connect(('squid', 3128)); s.close()\""]
|
||||
interval: 60s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 30s
|
||||
deploy:
|
||||
mode: global
|
||||
resources:
|
||||
limits:
|
||||
memory: 128M
|
||||
pids: 64
|
||||
reservations:
|
||||
memory: 64M
|
||||
restart_policy:
|
||||
condition: on-failure
|
||||
delay: 10s
|
||||
max_attempts: 3
|
||||
|
||||
networks:
|
||||
hermes_proxy:
|
||||
external: true
|
||||
default:
|
||||
external: true
|
||||
Reference in New Issue
Block a user