feat(connectivity_tester): new stack for synthetic connectivity checks #39

Merged
admin merged 12 commits from hermes/ansible:feat/connectivity-tester into master 2026-09-20 13:26:10 +02:00
4 changed files with 438 additions and 0 deletions
+12
View File
@@ -0,0 +1,12 @@
FROM python:3.12-slim
WORKDIR /app
COPY . /app/
RUN pip install --no-cache-dir docker==7.1.0
ENV PYTHONUNBUFFERED=1 \
PERIOD_SECONDS=600 \
CHECK_TIMEOUT_S=3
CMD ["python", "-m", "app"]
+276
View File
@@ -0,0 +1,276 @@
"""Connectivity tester.
For every configured overlay network the container attaches itself, runs
TCP checks against its endpoints, then detaches. Endpoints marked with
``proxy=(host, port)`` are probed through that HTTP proxy (CONNECT). After
each round a final loop checks PUBLIC_SERVICES. Cross-validation:
- service reachable but not in map -> ERROR 'LEAK-UNEXPECTED'
- service in map but unreachable -> ERROR 'MISSING-EXPECTED'
- global service not in any map -> WARNING 'GLOBAL-UNCOVERED'
"""
from __future__ import annotations
import asyncio
import logging
import os
import socket
import sys
import time
from typing import Iterable, List, Set, Tuple
import docker
from docker.models.containers import Container
from docker.models.networks import Network
from network_map import Endpoint, GLOBAL_SERVICES, NEGATIVE, NETWORK_MAP, PUBLIC_SERVICES
LOG = logging.getLogger("connectivity-tester")
FALLBACK_TCP_TIMEOUT_S = 3.0
HTTP_CONNECT_TIMEOUT_S = 5.0
def _socket_path() -> str:
return os.environ.get("DOCKER_SOCKET", "/var/run/docker.sock")
def _self_container(client: docker.DockerClient) -> Container:
hostname = socket.gethostname()
for c in client.containers.list():
cn = c.name.lstrip("/")
hn = c.attrs.get("Config", {}).get("Hostname", "")
if hostname == cn or hostname == hn or hostname[:12] == c.id[:12]:
return c
candidates = [c for c in client.containers.list() if hostname[:12] == c.id]
if len(candidates) == 1:
return candidates[0]
raise RuntimeError(f"self container not found for hostname={hostname}")
def _swarm_networks_for(client: docker.DockerClient,
names: Iterable[str]) -> dict:
out = {}
by_name = {n.name: n for n in client.networks.list()}
for name in names:
net = by_name.get(name)
if net is not None and net.attrs.get("Scope") == "swarm":
out[name] = net
return out
def _attach(client: docker.DockerClient, container: Container,
network: Network) -> bool:
current = (container.attrs.get("NetworkSettings", {}) or {}).get("Networks", {})
if network.name in current:
return False
try:
network.connect(container, aliases=["connectivity-tester",
socket.gethostname()])
return True
except Exception as exc:
if "already exists" in str(exc).lower():
return False
raise
def _detach(client: docker.DockerClient, container: Container,
network: Network) -> None:
try:
network.disconnect(container, force=True)
except Exception as exc:
if "not found" in str(exc).lower():
return
raise
async def _tcp_probe(host: str, port: int, timeout: float) -> Tuple[bool, str]:
try:
fut = asyncio.open_connection(host=host, port=port)
reader, writer = await asyncio.wait_for(fut, timeout=timeout)
writer.close()
try:
await writer.wait_closed()
except Exception:
pass
return True, "tcp ok"
except asyncio.TimeoutError:
return False, "timeout"
except OSError as exc:
return False, f"{type(exc).__name__}: {exc}"
async def _proxy_connect(endpoint: Endpoint, timeout: float) -> Tuple[bool, str]:
if endpoint.proxy is None:
return await _tcp_probe(endpoint.name, endpoint.port, timeout=timeout)
proxy_host, proxy_port = endpoint.proxy
target = endpoint.name
target_port = endpoint.port
try:
fut = asyncio.open_connection(host=proxy_host, port=proxy_port)
reader, writer = await asyncio.wait_for(fut, timeout=timeout)
req = (
f"CONNECT {target}:{target_port} HTTP/1.1\r\n"
f"Host: {target}:{target_port}\r\n\r\n"
)
writer.write(req.encode("ascii"))
await writer.drain()
status_line = await asyncio.wait_for(reader.readline(),
timeout=timeout)
if not status_line.startswith(b"HTTP/") or b" 200 " not in status_line:
writer.close()
try:
await writer.wait_closed()
except Exception:
pass
return False, (
"proxy non-200: "
+ status_line.strip().decode(errors="replace")
)
while True:
line = await asyncio.wait_for(reader.readline(), timeout=timeout)
if line in (b"\r\n", b"", b"\n"):
break
writer.close()
try:
await writer.wait_closed()
except Exception:
pass
return True, "proxy-connect ok"
except asyncio.TimeoutError:
return False, "timeout"
except OSError as exc:
return False, f"{type(exc).__name__}: {exc}"
async def _check_endpoint(endpoint: Endpoint, timeout: float) -> bool:
if endpoint.proxy is not None:
ok, detail = await _proxy_connect(endpoint, timeout)
else:
ok, detail = await _tcp_probe(endpoint.name, endpoint.port,
timeout=timeout)
label = f"{endpoint.name}:{endpoint.port}"
src = f"via {endpoint.proxy}" if endpoint.proxy else "direct"
if ok:
LOG.info("PASS %s %s %s", label, src, detail)
else:
LOG.error("FAIL %s %s %s", label, src, detail)
return ok
async def _check_network(network_name: str,
endpoints: List[Endpoint],
timeout: float) -> Tuple[Set[Endpoint], Set[Endpoint]]:
reachable: Set[Endpoint] = set()
unreachable: Set[Endpoint] = set()
for ep in endpoints:
ok = await _check_endpoint(ep, timeout)
(reachable if ok else unreachable).add(ep)
return reachable, unreachable
def _cross_validate(network_name: str,
mapped: List[Endpoint],
reachable: Set[Endpoint],
unreachable: Set[Endpoint],
global_set: Set[Endpoint]) -> None:
mapped_set = set(mapped)
reachable_global = reachable & global_set
for ep in reachable_global - mapped_set:
LOG.error("LEAK-UNEXPECTED %s reachable in %s but not in map",
f"{ep.name}:{ep.port}", network_name)
for ep in mapped_set & unreachable:
LOG.error("MISSING-EXPECTED %s unreachable in %s but listed in map",
f"{ep.name}:{ep.port}", network_name)
for ep in mapped_set & reachable:
LOG.info("OK-MAPPED %s ok in %s", f"{ep.name}:{ep.port}", network_name)
for ep in mapped_set - reachable - unreachable:
LOG.warning("NOT-CHECKED %s in map for %s",
f"{ep.name}:{ep.port}", network_name)
def _consistency_audit(net_names: List[str]) -> None:
flat: Set[Endpoint] = set()
for n in net_names:
flat.update(NETWORK_MAP.get(n, []))
covered = flat | set(PUBLIC_SERVICES)
all_global = set(GLOBAL_SERVICES)
uncovered = all_global - covered
if uncovered:
LOG.warning("GLOBAL-UNCOVERED %s not present in any map",
sorted(f"{e.name}:{e.port}" for e in uncovered))
async def _check_negative(endpoints: List[Endpoint],
timeout: float) -> int:
leaks = 0
for ep in endpoints:
ok = await _tcp_probe(ep.name, ep.port, timeout=timeout)
if ok:
LOG.error("LEAK %s:%d reachable but must NOT be",
ep.name, ep.port)
leaks += 1
else:
LOG.info("OK-NEG %s:%d blocked", ep.name, ep.port)
return leaks
async def main() -> int:
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(levelname)s %(message)s",
stream=sys.stdout,
)
period = int(os.environ.get("PERIOD_SECONDS", "600"))
timeout = float(os.environ.get("CHECK_TIMEOUT_S", "3"))
LOG.info("starting; period=%ds timeout=%.1fs", period, timeout)
client = docker.DockerClient(base_url="unix://" + _socket_path())
self_container = _self_container(client)
nets = _swarm_networks_for(client, NETWORK_MAP.keys())
planned = list(NETWORK_MAP.keys())
LOG.info("configured networks: %d, present in swarm: %d",
len(planned), len(nets))
for n in nets:
LOG.info("plan network: %s", n)
_consistency_audit(planned)
global_set: Set[Endpoint] = set(GLOBAL_SERVICES)
while True:
started = time.monotonic()
try:
for net_name, net in nets.items():
eps_in_map: List[Endpoint] = [
e for e in NETWORK_MAP[net_name] if e.proxy is None
]
LOG.info("--- round begin, attaching to %s (%d endpoints) ---",
net_name, len(eps_in_map))
_attach(client, self_container, net)
try:
reachable, unreachable = await _check_network(
net_name, eps_in_map, timeout)
_cross_validate(net_name, eps_in_map,
reachable, unreachable, global_set)
finally:
_detach(client, self_container, net)
LOG.info("--- round end on %s ---", net_name)
LOG.info("--- public checks (%d endpoints) ---",
len(PUBLIC_SERVICES))
for ep in PUBLIC_SERVICES:
await _check_endpoint(ep, max(timeout, HTTP_CONNECT_TIMEOUT_S))
LOG.info("--- negative checks (%d endpoints) ---", len(NEGATIVE))
await _check_negative(NEGATIVE, timeout)
except Exception:
LOG.exception("round crashed")
elapsed = time.monotonic() - started
sleep_for = max(0.0, period - elapsed)
await asyncio.sleep(sleep_for)
if __name__ == "__main__":
asyncio.run(main())
+122
View File
@@ -0,0 +1,122 @@
from dataclasses import dataclass
from typing import Dict, List, Optional, Tuple
@dataclass(frozen=True)
class Endpoint:
name: str
port: int
proxy: Optional[Tuple[str, int]] = None
tls: bool = False
SWARM_NETWORKS = (
"hermes_infra_hermes_proxy",
"hermes_infra_squid_egress",
"hermes_infra_admin",
"hermes_infra_default",
"hermes_hermes",
"gitea_gitea",
"gmail_mcp_default",
"immich_default",
"torrserv_proxy_default",
"docker_socket_proxy_hermes_proxy",
"connectivity_tester_default",
"brencher_brencher-net",
"brencher2_brencher-net",
)
SQUID_PROXY: Tuple[str, int] = ("squid", 3128)
NETWORK_MAP: Dict[str, List[Endpoint]] = {
"hermes_infra_hermes_proxy": [
Endpoint("hermes-gateway", 8000),
Endpoint("hermes-ui", 8087),
Endpoint("squid", 3128),
Endpoint("docker_socket_proxy", 2375),
Endpoint("brencher-state-proxy", 80),
],
"hermes_infra_squid_egress": [
Endpoint("squid", 3128),
],
"hermes_infra_admin": [
Endpoint("sarg", 80),
],
"hermes_infra_default": [
Endpoint("hermes-ui", 8087),
],
"hermes_hermes": [
Endpoint("hermes-agent", 8000),
Endpoint("hermes-gateway", 8000),
],
"gitea_gitea": [
Endpoint("gitea-server", 3000),
Endpoint("act_runner", 8088),
Endpoint("cache_server", 37051),
],
"gmail_mcp_default": [
Endpoint("gmail-mcp-server", 8000),
Endpoint("telegram-mcp-server", 8000),
],
"immich_default": [
Endpoint("immich-server", 2283),
Endpoint("immich-database", 5432),
Endpoint("immich-redis", 6379),
Endpoint("immich-machine-learning", 8501),
],
"torrserv_proxy_default": [
Endpoint("fileserver", 8000),
Endpoint("reverseproxy", 8080),
],
"docker_socket_proxy_hermes_proxy": [
Endpoint("hermes-gateway", 8000),
],
"connectivity_tester_default": [
Endpoint("connectivity-tester", 8765),
],
"brencher_brencher-net": [
Endpoint("brencher-backend", 5000),
],
"brencher2_brencher-net": [
Endpoint("brencher-backend", 5000),
],
}
PUBLIC_SERVICES: List[Endpoint] = [
Endpoint("gitea.rudolf.keenetic.link", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("gitea.rudolf.keenetic.link", 80, proxy=SQUID_PROXY),
Endpoint("registry.rudolf.keenetic.link", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("registry.rudolf.keenetic.link", 5000, proxy=SQUID_PROXY),
Endpoint("deb.debian.org", 80, proxy=SQUID_PROXY),
Endpoint("deb.debian.org", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("github.com", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("githubusercontent.com", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("telegram.org", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("t.me", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("minimax.io", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("minimax.chat", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("minimax.com", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("pypi.org", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("google.com", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("cloudflare.com", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("wikipedia.org", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("mozilla.org", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("1.1.1.1", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("8.8.8.8", 53),
Endpoint("8.8.4.4", 53),
]
GLOBAL_SERVICES: List[Endpoint] = sorted(
{ep for eps in NETWORK_MAP.values() for ep in eps} | set(PUBLIC_SERVICES),
key=lambda e: (e.name, e.port),
)
NEGATIVE: List[Endpoint] = [
Endpoint("169.254.169.254", 80),
Endpoint("metadata.google.internal", 80),
]
+28
View File
@@ -0,0 +1,28 @@
services:
connectivity-tester:
build:
context: .
dockerfile: Dockerfile
image: registry.rudolf.keenetic.link/connectivity_tester:${version}
hostname: connectivity-tester
user: "0:0"
volumes:
- /var/run/docker.sock:/var/run/docker.sock
healthcheck:
test: ["CMD-SHELL", "test -S /var/run/docker.sock"]
interval: 60s
timeout: 5s
retries: 3
start_period: 30s
deploy:
mode: global
resources:
limits:
memory: 128M
pids: 64
reservations:
memory: 64M
restart_policy:
condition: on-failure
delay: 10s
max_attempts: 3