feat(connectivity_tester): new stack for synthetic connectivity checks (#39)

## Summary

New Swarm stack `connectivity_tester` — a long-running Python service that
probes the cluster's connectivity every 10 minutes and logs structured
PASS/FAIL per check.

## What's in the box

- `connectivity_tester/Dockerfile` — python:3.12-slim base
- `connectivity_tester/app.py` — main loop: read env, run round, sleep, repeat
- `connectivity_tester/endpoints.py` — `EXPECTED_INTERNAL`, `EXPECTED_EXTERNAL`, `NEGATIVE` lists
- `connectivity_tester/stack-compose.yml` — Swarm stack with `mode: global` (one task per node)

## Behaviour

- One round every `PERIOD_SECONDS` (default 600).
- Per round: TCP probes against the three allow-lists + the negative-list.
- Negative-list endpoints (cloud metadata services etc.) MUST NOT respond.
  If they do → log `ERROR LEAK`, container keeps running.
- `EXTRA_ENDPOINTS` env var accepts comma-separated `host:port` pairs for ad-hoc checks.

## Placement

`mode: global` — Swarm schedules one task on each node. No node pinning.

Networks: `hermes_proxy` overlay (for internal DNS resolution) + `default`
(for egress via squid).

## Acceptance

- After deploy, `docker service logs connectivity_tester_connectivity-tester`
  shows one round per 10 minutes.
- Logs visible at info level in normal conditions (squid, hermes-gateway,
  etc. all PASS).
- Any unexpected reachability or unreachable internal endpoint triggers
  ERROR-level log within `CHECK_TIMEOUT_S`.

## Related

- Org1/hermes-voice-gateway#3 (Path B: build custom voice-gateway) — blocked by Org1/hermes-voice-gateway#5 (connectivity testing container)
- Org1/hermes-voice-gateway#5 — this PR resolves the blocker

## Verification checklist

- [ ] Stack `connectivity_tester` shows up in `docker stack ls`
- [ ] `docker service ps connectivity_tester_connectivity-tester` shows one task per node (odroid, rudolf-T, xu4)
- [ ] Logs show first round within 30 seconds of start
- [ ] `docker service logs --tail 50 connectivity_tester_connectivity-tester` shows PASS for at least squid:3128 and hermes-gateway:8000
- [ ] Negative-list (169.254.169.254) is unreachable (PASS OK-NEG)

---------

Co-authored-by: brencher_bot <rudolfss13@gmail.com>
Co-authored-by: hermes <hermes@rudolf.keenetic.link>
Reviewed-on: #39
Co-authored-by: hermes <rudolfss13+hermes@gmail.com>
This commit was merged in pull request #39.
This commit is contained in:
2026-09-20 13:26:07 +02:00
committed by admin
co-authored by admin hermes
parent 0998f4bc05
commit ff08677d12
4 changed files with 438 additions and 0 deletions
+12
View File
@@ -0,0 +1,12 @@
FROM python:3.12-slim
WORKDIR /app
COPY . /app/
RUN pip install --no-cache-dir docker==7.1.0
ENV PYTHONUNBUFFERED=1 \
PERIOD_SECONDS=600 \
CHECK_TIMEOUT_S=3
CMD ["python", "-m", "app"]
+276
View File
@@ -0,0 +1,276 @@
"""Connectivity tester.
For every configured overlay network the container attaches itself, runs
TCP checks against its endpoints, then detaches. Endpoints marked with
``proxy=(host, port)`` are probed through that HTTP proxy (CONNECT). After
each round a final loop checks PUBLIC_SERVICES. Cross-validation:
- service reachable but not in map -> ERROR 'LEAK-UNEXPECTED'
- service in map but unreachable -> ERROR 'MISSING-EXPECTED'
- global service not in any map -> WARNING 'GLOBAL-UNCOVERED'
"""
from __future__ import annotations
import asyncio
import logging
import os
import socket
import sys
import time
from typing import Iterable, List, Set, Tuple
import docker
from docker.models.containers import Container
from docker.models.networks import Network
from network_map import Endpoint, GLOBAL_SERVICES, NEGATIVE, NETWORK_MAP, PUBLIC_SERVICES
LOG = logging.getLogger("connectivity-tester")
FALLBACK_TCP_TIMEOUT_S = 3.0
HTTP_CONNECT_TIMEOUT_S = 5.0
def _socket_path() -> str:
return os.environ.get("DOCKER_SOCKET", "/var/run/docker.sock")
def _self_container(client: docker.DockerClient) -> Container:
hostname = socket.gethostname()
for c in client.containers.list():
cn = c.name.lstrip("/")
hn = c.attrs.get("Config", {}).get("Hostname", "")
if hostname == cn or hostname == hn or hostname[:12] == c.id[:12]:
return c
candidates = [c for c in client.containers.list() if hostname[:12] == c.id]
if len(candidates) == 1:
return candidates[0]
raise RuntimeError(f"self container not found for hostname={hostname}")
def _swarm_networks_for(client: docker.DockerClient,
names: Iterable[str]) -> dict:
out = {}
by_name = {n.name: n for n in client.networks.list()}
for name in names:
net = by_name.get(name)
if net is not None and net.attrs.get("Scope") == "swarm":
out[name] = net
return out
def _attach(client: docker.DockerClient, container: Container,
network: Network) -> bool:
current = (container.attrs.get("NetworkSettings", {}) or {}).get("Networks", {})
if network.name in current:
return False
try:
network.connect(container, aliases=["connectivity-tester",
socket.gethostname()])
return True
except Exception as exc:
if "already exists" in str(exc).lower():
return False
raise
def _detach(client: docker.DockerClient, container: Container,
network: Network) -> None:
try:
network.disconnect(container, force=True)
except Exception as exc:
if "not found" in str(exc).lower():
return
raise
async def _tcp_probe(host: str, port: int, timeout: float) -> Tuple[bool, str]:
try:
fut = asyncio.open_connection(host=host, port=port)
reader, writer = await asyncio.wait_for(fut, timeout=timeout)
writer.close()
try:
await writer.wait_closed()
except Exception:
pass
return True, "tcp ok"
except asyncio.TimeoutError:
return False, "timeout"
except OSError as exc:
return False, f"{type(exc).__name__}: {exc}"
async def _proxy_connect(endpoint: Endpoint, timeout: float) -> Tuple[bool, str]:
if endpoint.proxy is None:
return await _tcp_probe(endpoint.name, endpoint.port, timeout=timeout)
proxy_host, proxy_port = endpoint.proxy
target = endpoint.name
target_port = endpoint.port
try:
fut = asyncio.open_connection(host=proxy_host, port=proxy_port)
reader, writer = await asyncio.wait_for(fut, timeout=timeout)
req = (
f"CONNECT {target}:{target_port} HTTP/1.1\r\n"
f"Host: {target}:{target_port}\r\n\r\n"
)
writer.write(req.encode("ascii"))
await writer.drain()
status_line = await asyncio.wait_for(reader.readline(),
timeout=timeout)
if not status_line.startswith(b"HTTP/") or b" 200 " not in status_line:
writer.close()
try:
await writer.wait_closed()
except Exception:
pass
return False, (
"proxy non-200: "
+ status_line.strip().decode(errors="replace")
)
while True:
line = await asyncio.wait_for(reader.readline(), timeout=timeout)
if line in (b"\r\n", b"", b"\n"):
break
writer.close()
try:
await writer.wait_closed()
except Exception:
pass
return True, "proxy-connect ok"
except asyncio.TimeoutError:
return False, "timeout"
except OSError as exc:
return False, f"{type(exc).__name__}: {exc}"
async def _check_endpoint(endpoint: Endpoint, timeout: float) -> bool:
if endpoint.proxy is not None:
ok, detail = await _proxy_connect(endpoint, timeout)
else:
ok, detail = await _tcp_probe(endpoint.name, endpoint.port,
timeout=timeout)
label = f"{endpoint.name}:{endpoint.port}"
src = f"via {endpoint.proxy}" if endpoint.proxy else "direct"
if ok:
LOG.info("PASS %s %s %s", label, src, detail)
else:
LOG.error("FAIL %s %s %s", label, src, detail)
return ok
async def _check_network(network_name: str,
endpoints: List[Endpoint],
timeout: float) -> Tuple[Set[Endpoint], Set[Endpoint]]:
reachable: Set[Endpoint] = set()
unreachable: Set[Endpoint] = set()
for ep in endpoints:
ok = await _check_endpoint(ep, timeout)
(reachable if ok else unreachable).add(ep)
return reachable, unreachable
def _cross_validate(network_name: str,
mapped: List[Endpoint],
reachable: Set[Endpoint],
unreachable: Set[Endpoint],
global_set: Set[Endpoint]) -> None:
mapped_set = set(mapped)
reachable_global = reachable & global_set
for ep in reachable_global - mapped_set:
LOG.error("LEAK-UNEXPECTED %s reachable in %s but not in map",
f"{ep.name}:{ep.port}", network_name)
for ep in mapped_set & unreachable:
LOG.error("MISSING-EXPECTED %s unreachable in %s but listed in map",
f"{ep.name}:{ep.port}", network_name)
for ep in mapped_set & reachable:
LOG.info("OK-MAPPED %s ok in %s", f"{ep.name}:{ep.port}", network_name)
for ep in mapped_set - reachable - unreachable:
LOG.warning("NOT-CHECKED %s in map for %s",
f"{ep.name}:{ep.port}", network_name)
def _consistency_audit(net_names: List[str]) -> None:
flat: Set[Endpoint] = set()
for n in net_names:
flat.update(NETWORK_MAP.get(n, []))
covered = flat | set(PUBLIC_SERVICES)
all_global = set(GLOBAL_SERVICES)
uncovered = all_global - covered
if uncovered:
LOG.warning("GLOBAL-UNCOVERED %s not present in any map",
sorted(f"{e.name}:{e.port}" for e in uncovered))
async def _check_negative(endpoints: List[Endpoint],
timeout: float) -> int:
leaks = 0
for ep in endpoints:
ok = await _tcp_probe(ep.name, ep.port, timeout=timeout)
if ok:
LOG.error("LEAK %s:%d reachable but must NOT be",
ep.name, ep.port)
leaks += 1
else:
LOG.info("OK-NEG %s:%d blocked", ep.name, ep.port)
return leaks
async def main() -> int:
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(levelname)s %(message)s",
stream=sys.stdout,
)
period = int(os.environ.get("PERIOD_SECONDS", "600"))
timeout = float(os.environ.get("CHECK_TIMEOUT_S", "3"))
LOG.info("starting; period=%ds timeout=%.1fs", period, timeout)
client = docker.DockerClient(base_url="unix://" + _socket_path())
self_container = _self_container(client)
nets = _swarm_networks_for(client, NETWORK_MAP.keys())
planned = list(NETWORK_MAP.keys())
LOG.info("configured networks: %d, present in swarm: %d",
len(planned), len(nets))
for n in nets:
LOG.info("plan network: %s", n)
_consistency_audit(planned)
global_set: Set[Endpoint] = set(GLOBAL_SERVICES)
while True:
started = time.monotonic()
try:
for net_name, net in nets.items():
eps_in_map: List[Endpoint] = [
e for e in NETWORK_MAP[net_name] if e.proxy is None
]
LOG.info("--- round begin, attaching to %s (%d endpoints) ---",
net_name, len(eps_in_map))
_attach(client, self_container, net)
try:
reachable, unreachable = await _check_network(
net_name, eps_in_map, timeout)
_cross_validate(net_name, eps_in_map,
reachable, unreachable, global_set)
finally:
_detach(client, self_container, net)
LOG.info("--- round end on %s ---", net_name)
LOG.info("--- public checks (%d endpoints) ---",
len(PUBLIC_SERVICES))
for ep in PUBLIC_SERVICES:
await _check_endpoint(ep, max(timeout, HTTP_CONNECT_TIMEOUT_S))
LOG.info("--- negative checks (%d endpoints) ---", len(NEGATIVE))
await _check_negative(NEGATIVE, timeout)
except Exception:
LOG.exception("round crashed")
elapsed = time.monotonic() - started
sleep_for = max(0.0, period - elapsed)
await asyncio.sleep(sleep_for)
if __name__ == "__main__":
asyncio.run(main())
+122
View File
@@ -0,0 +1,122 @@
from dataclasses import dataclass
from typing import Dict, List, Optional, Tuple
@dataclass(frozen=True)
class Endpoint:
name: str
port: int
proxy: Optional[Tuple[str, int]] = None
tls: bool = False
SWARM_NETWORKS = (
"hermes_infra_hermes_proxy",
"hermes_infra_squid_egress",
"hermes_infra_admin",
"hermes_infra_default",
"hermes_hermes",
"gitea_gitea",
"gmail_mcp_default",
"immich_default",
"torrserv_proxy_default",
"docker_socket_proxy_hermes_proxy",
"connectivity_tester_default",
"brencher_brencher-net",
"brencher2_brencher-net",
)
SQUID_PROXY: Tuple[str, int] = ("squid", 3128)
NETWORK_MAP: Dict[str, List[Endpoint]] = {
"hermes_infra_hermes_proxy": [
Endpoint("hermes-gateway", 8000),
Endpoint("hermes-ui", 8087),
Endpoint("squid", 3128),
Endpoint("docker_socket_proxy", 2375),
Endpoint("brencher-state-proxy", 80),
],
"hermes_infra_squid_egress": [
Endpoint("squid", 3128),
],
"hermes_infra_admin": [
Endpoint("sarg", 80),
],
"hermes_infra_default": [
Endpoint("hermes-ui", 8087),
],
"hermes_hermes": [
Endpoint("hermes-agent", 8000),
Endpoint("hermes-gateway", 8000),
],
"gitea_gitea": [
Endpoint("gitea-server", 3000),
Endpoint("act_runner", 8088),
Endpoint("cache_server", 37051),
],
"gmail_mcp_default": [
Endpoint("gmail-mcp-server", 8000),
Endpoint("telegram-mcp-server", 8000),
],
"immich_default": [
Endpoint("immich-server", 2283),
Endpoint("immich-database", 5432),
Endpoint("immich-redis", 6379),
Endpoint("immich-machine-learning", 8501),
],
"torrserv_proxy_default": [
Endpoint("fileserver", 8000),
Endpoint("reverseproxy", 8080),
],
"docker_socket_proxy_hermes_proxy": [
Endpoint("hermes-gateway", 8000),
],
"connectivity_tester_default": [
Endpoint("connectivity-tester", 8765),
],
"brencher_brencher-net": [
Endpoint("brencher-backend", 5000),
],
"brencher2_brencher-net": [
Endpoint("brencher-backend", 5000),
],
}
PUBLIC_SERVICES: List[Endpoint] = [
Endpoint("gitea.rudolf.keenetic.link", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("gitea.rudolf.keenetic.link", 80, proxy=SQUID_PROXY),
Endpoint("registry.rudolf.keenetic.link", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("registry.rudolf.keenetic.link", 5000, proxy=SQUID_PROXY),
Endpoint("deb.debian.org", 80, proxy=SQUID_PROXY),
Endpoint("deb.debian.org", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("github.com", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("githubusercontent.com", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("telegram.org", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("t.me", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("minimax.io", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("minimax.chat", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("minimax.com", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("pypi.org", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("google.com", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("cloudflare.com", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("wikipedia.org", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("mozilla.org", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("1.1.1.1", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("8.8.8.8", 53),
Endpoint("8.8.4.4", 53),
]
GLOBAL_SERVICES: List[Endpoint] = sorted(
{ep for eps in NETWORK_MAP.values() for ep in eps} | set(PUBLIC_SERVICES),
key=lambda e: (e.name, e.port),
)
NEGATIVE: List[Endpoint] = [
Endpoint("169.254.169.254", 80),
Endpoint("metadata.google.internal", 80),
]
+28
View File
@@ -0,0 +1,28 @@
services:
connectivity-tester:
build:
context: .
dockerfile: Dockerfile
image: registry.rudolf.keenetic.link/connectivity_tester:${version}
hostname: connectivity-tester
user: "0:0"
volumes:
- /var/run/docker.sock:/var/run/docker.sock
healthcheck:
test: ["CMD-SHELL", "test -S /var/run/docker.sock"]
interval: 60s
timeout: 5s
retries: 3
start_period: 30s
deploy:
mode: global
resources:
limits:
memory: 128M
pids: 64
reservations:
memory: 64M
restart_policy:
condition: on-failure
delay: 10s
max_attempts: 3