feat(connectivity_tester): Endpoint dataclass, proxy-aware checks, public hosts

Replace tuple-of-strings with frozen Endpoint dataclass carrying optional
(proxy_host, proxy_port) tuple. Add network_map.NETWORK_MAP populated from
the live swarm endpoints discovered via the docker socket, and a separate
PUBLIC_SERVICES list sourced from the squid allowlist plus well-known
public hosts. Endpoints with proxy are checked via HTTP CONNECT through
that proxy; endpoints without are probed directly via TCP after attaching
to the relevant overlay network. Cross-validation rules cover leaks and
missing expectations per round.
This commit is contained in:
2026-09-15 10:11:49 +00:00
parent d5b1a23bcb
commit 1e9d33485d
2 changed files with 269 additions and 132 deletions
+155 -100
View File
@@ -1,10 +1,12 @@
"""Connectivity tester.
For every network in network_map.NETWORK_MAP the container attaches itself,
runs TCP checks against the listed services, detaches, and proceeds to the
next network. Cross-validates GLOBAL_SERVICES against each network map: logs
errors when reachable services are absent from the map or when expected
services are unreachable.
For every configured overlay network the container attaches itself, runs
TCP checks against its endpoints, then detaches. Endpoints marked with
``proxy=(host, port)`` are probed through that HTTP proxy (CONNECT). After
each round a final loop checks PUBLIC_SERVICES. Cross-validation:
- service reachable but not in map -> ERROR 'LEAK-UNEXPECTED'
- service in map but unreachable -> ERROR 'MISSING-EXPECTED'
- global service not in any map -> WARNING 'GLOBAL-UNCOVERED'
"""
from __future__ import annotations
@@ -20,10 +22,13 @@ import docker
from docker.models.containers import Container
from docker.models.networks import Network
from network_map import GLOBAL_SERVICES, NETWORK_MAP, Service
from network_map import Endpoint, GLOBAL_SERVICES, NEGATIVE, NETWORK_MAP, PUBLIC_SERVICES
LOG = logging.getLogger("connectivity-tester")
FALLBACK_TCP_TIMEOUT_S = 3.0
HTTP_CONNECT_TIMEOUT_S = 5.0
def _socket_path() -> str:
return os.environ.get("DOCKER_SOCKET", "/var/run/docker.sock")
@@ -32,55 +37,39 @@ def _socket_path() -> str:
def _self_container(client: docker.DockerClient) -> Container:
hostname = socket.gethostname()
for c in client.containers.list():
if (c.attrs.get("Config", {}).get("Hostname") == hostname
or c.name.lstrip("/") == hostname):
cn = c.name.lstrip("/")
hn = c.attrs.get("Config", {}).get("Hostname", "")
if hostname == cn or hostname == hn or hostname[:12] == c.id[:12]:
return c
candidates = [c for c in client.containers.list()
if hostname[:12] in c.id]
candidates = [c for c in client.containers.list() if hostname[:12] == c.id]
if len(candidates) == 1:
return candidates[0]
raise RuntimeError(f"self container not found for hostname={hostname}")
def _filtered_networks(client: docker.DockerClient) -> List[Network]:
keep_prefixes = (
"hermes_infra_", "gitea_", "gmail_mcp_", "telegram_",
"hermes_", "brencher", "registry_", "immich_",
"torrserv_proxy_", "photosHelper_", "nginx_",
"docker_socket_proxy_",
)
out = []
for n in client.networks.list():
if n.attrs.get("Scope") != "swarm":
continue
name = n.name
if name in ("ingress",):
continue
if any(name.startswith(p) for p in keep_prefixes):
out.append(n)
return out
def _build_map_from_running(networks: Iterable[Network]) -> dict:
"""Reverse-engineer NAME -> swarm network name from running container endpoints."""
def _swarm_networks_for(client: docker.DockerClient,
names: Iterable[str]) -> dict:
out = {}
for net in networks:
endpoint_count = len(net.attrs.get("Containers") or {})
out[net.name] = {"endpoints": endpoint_count, "id": net.id}
by_name = {n.name: n for n in client.networks.list()}
for name in names:
net = by_name.get(name)
if net is not None and net.attrs.get("Scope") == "swarm":
out[name] = net
return out
def _attach(client: docker.DockerClient, container: Container,
network: Network) -> None:
if network.name in (container.attrs.get("NetworkSettings", {})
.get("Networks", {}).keys()):
return
network: Network) -> bool:
current = (container.attrs.get("NetworkSettings", {}) or {}).get("Networks", {})
if network.name in current:
return False
try:
network.connect(container, aliases=["connectivity-tester",
socket.gethostname()])
return True
except Exception as exc:
if "already exists" in str(exc).lower():
return
return False
raise
@@ -94,9 +83,9 @@ def _detach(client: docker.DockerClient, container: Container,
raise
async def _tcp_check(host: str, port: int, timeout: float) -> Tuple[bool, str]:
async def _tcp_probe(host: str, port: int, timeout: float) -> Tuple[bool, str]:
try:
fut = asyncio.open_connection(host, port)
fut = asyncio.open_connection(host=host, port=port)
reader, writer = await asyncio.wait_for(fut, timeout=timeout)
writer.close()
try:
@@ -110,60 +99,119 @@ async def _tcp_check(host: str, port: int, timeout: float) -> Tuple[bool, str]:
return False, f"{type(exc).__name__}: {exc}"
async def _check_one(host: str, port: int, label: str,
timeout: float) -> bool:
ok, detail = await _tcp_check(host, port, timeout)
if ok:
LOG.info("PASS %s:%d (%s) %s", host, port, label, detail)
async def _proxy_connect(endpoint: Endpoint, timeout: float) -> Tuple[bool, str]:
if endpoint.proxy is None:
return await _tcp_probe(endpoint.name, endpoint.port, timeout=timeout)
proxy_host, proxy_port = endpoint.proxy
target = endpoint.name
target_port = endpoint.port
try:
fut = asyncio.open_connection(host=proxy_host, port=proxy_port)
reader, writer = await asyncio.wait_for(fut, timeout=timeout)
req = (
f"CONNECT {target}:{target_port} HTTP/1.1\r\n"
f"Host: {target}:{target_port}\r\n\r\n"
)
writer.write(req.encode("ascii"))
await writer.drain()
status_line = await asyncio.wait_for(reader.readline(),
timeout=timeout)
if not status_line.startswith(b"HTTP/") or b" 200 " not in status_line:
writer.close()
try:
await writer.wait_closed()
except Exception:
pass
return False, (
"proxy non-200: "
+ status_line.strip().decode(errors="replace")
)
while True:
line = await asyncio.wait_for(reader.readline(), timeout=timeout)
if line in (b"\r\n", b"", b"\n"):
break
writer.close()
try:
await writer.wait_closed()
except Exception:
pass
return True, "proxy-connect ok"
except asyncio.TimeoutError:
return False, "timeout"
except OSError as exc:
return False, f"{type(exc).__name__}: {exc}"
async def _check_endpoint(endpoint: Endpoint, timeout: float) -> bool:
if endpoint.proxy is not None:
ok, detail = await _proxy_connect(endpoint, timeout)
else:
LOG.error("FAIL %s:%d (%s) %s", host, port, label, detail)
ok, detail = await _tcp_probe(endpoint.name, endpoint.port,
timeout=timeout)
label = f"{endpoint.name}:{endpoint.port}"
src = f"via {endpoint.proxy}" if endpoint.proxy else "direct"
if ok:
LOG.info("PASS %s %s %s", label, src, detail)
else:
LOG.error("FAIL %s %s %s", label, src, detail)
return ok
async def _check_network(network: Network, services: List[Service],
timeout: float) -> Tuple[Set[Service], Set[Service]]:
reachable: Set[Service] = set()
unreachable: Set[Service] = set()
for host, port in services:
ok = await _check_one(host, port, network.name, timeout)
(reachable if ok else unreachable).add((host, port))
async def _check_network(network_name: str,
endpoints: List[Endpoint],
timeout: float) -> Tuple[Set[Endpoint], Set[Endpoint]]:
reachable: Set[Endpoint] = set()
unreachable: Set[Endpoint] = set()
for ep in endpoints:
ok = await _check_endpoint(ep, timeout)
(reachable if ok else unreachable).add(ep)
return reachable, unreachable
def _discover_attached(client: docker.DockerClient,
container: Container) -> Set[str]:
return set((container.attrs.get("NetworkSettings", {}) or {})
.get("Networks", {}).keys())
def _cross_validate(network_name: str,
mapped: List[Service],
reachable: Set[Service],
unreachable: Set[Service],
all_global: List[Service]) -> None:
mapped: List[Endpoint],
reachable: Set[Endpoint],
unreachable: Set[Endpoint],
global_set: Set[Endpoint]) -> None:
mapped_set = set(mapped)
reachable_global = reachable & set(all_global)
for svc in reachable_global - mapped_set:
reachable_global = reachable & global_set
for ep in reachable_global - mapped_set:
LOG.error("LEAK-UNEXPECTED %s reachable in %s but not in map",
f"{svc[0]}:{svc[1]}", network_name)
for svc in mapped_set & unreachable:
f"{ep.name}:{ep.port}", network_name)
for ep in mapped_set & unreachable:
LOG.error("MISSING-EXPECTED %s unreachable in %s but listed in map",
f"{svc[0]}:{svc[1]}", network_name)
for svc in mapped_set & reachable:
LOG.info("OK-MAPPED %s ok in %s", f"{svc[0]}:{svc[1]}", network_name)
for svc in mapped_set - reachable - unreachable:
f"{ep.name}:{ep.port}", network_name)
for ep in mapped_set & reachable:
LOG.info("OK-MAPPED %s ok in %s", f"{ep.name}:{ep.port}", network_name)
for ep in mapped_set - reachable - unreachable:
LOG.warning("NOT-CHECKED %s in map for %s",
f"{svc[0]}:{svc[1]}", network_name)
f"{ep.name}:{ep.port}", network_name)
def _consistency_audit(networks_names: List[str], all_global: List[Service]) -> None:
flat: Set[Service] = set()
for n in networks_names:
def _consistency_audit(net_names: List[str]) -> None:
flat: Set[Endpoint] = set()
for n in net_names:
flat.update(NETWORK_MAP.get(n, []))
leftover = set(all_global) - flat
if leftover:
covered = flat | set(PUBLIC_SERVICES)
all_global = set(GLOBAL_SERVICES)
uncovered = all_global - covered
if uncovered:
LOG.warning("GLOBAL-UNCOVERED %s not present in any map",
sorted(f"{s[0]}:{s[1]}" for s in leftover))
sorted(f"{e.name}:{e.port}" for e in uncovered))
async def _check_negative(endpoints: List[Endpoint],
timeout: float) -> int:
leaks = 0
for ep in endpoints:
ok = await _tcp_probe(ep.name, ep.port, timeout=timeout)
if ok:
LOG.error("LEAK %s:%d reachable but must NOT be",
ep.name, ep.port)
leaks += 1
else:
LOG.info("OK-NEG %s:%d blocked", ep.name, ep.port)
return leaks
async def main() -> int:
@@ -178,37 +226,44 @@ async def main() -> int:
client = docker.DockerClient(base_url="unix://" + _socket_path())
self_container = _self_container(client)
self_id_short = self_container.id[:12]
networks_docker = _filtered_networks(client)
networks_known = [n for n in networks_docker if n.name in NETWORK_MAP]
nets = _swarm_networks_for(client, NETWORK_MAP.keys())
planned = list(NETWORK_MAP.keys())
LOG.info("configured networks: %d, present in swarm: %d",
len(NETWORK_MAP), len(networks_known))
for n in networks_known:
LOG.info("plan network: %s", n.name)
len(planned), len(nets))
for n in nets:
LOG.info("plan network: %s", n)
_consistency_audit([n.name for n in networks_known], GLOBAL_SERVICES)
_consistency_audit(planned)
global_set: Set[Endpoint] = set(GLOBAL_SERVICES)
while True:
started = time.monotonic()
failures = 0
try:
for network in networks_known:
LOG.info("--- round begin, attaching to %s ---", network.name)
_attach(client, self_container, network)
for net_name, net in nets.items():
eps_in_map: List[Endpoint] = [
e for e in NETWORK_MAP[net_name] if e.proxy is None
]
LOG.info("--- round begin, attaching to %s (%d endpoints) ---",
net_name, len(eps_in_map))
_attach(client, self_container, net)
try:
reachable, unreachable = await _check_network(
network, NETWORK_MAP[network.name], timeout)
_cross_validate(
network.name,
NETWORK_MAP[network.name],
reachable,
unreachable,
GLOBAL_SERVICES,
)
net_name, eps_in_map, timeout)
_cross_validate(net_name, eps_in_map,
reachable, unreachable, global_set)
finally:
_detach(client, self_container, network)
LOG.info("--- round end on %s ---", network.name)
_detach(client, self_container, net)
LOG.info("--- round end on %s ---", net_name)
LOG.info("--- public checks (%d endpoints) ---",
len(PUBLIC_SERVICES))
for ep in PUBLIC_SERVICES:
await _check_endpoint(ep, max(timeout, HTTP_CONNECT_TIMEOUT_S))
LOG.info("--- negative checks (%d endpoints) ---", len(NEGATIVE))
await _check_negative(NEGATIVE, timeout)
except Exception:
LOG.exception("round crashed")
+114 -32
View File
@@ -1,40 +1,122 @@
from typing import Dict, List, Tuple
from dataclasses import dataclass
from typing import Dict, List, Optional, Tuple
Service = Tuple[str, int]
GLOBAL_SERVICES: List[Service] = [
("hermes-gateway", 8000),
("hermes-agent", 8000),
("hermes-ui", 8087),
("squid", 3128),
("gitea-server", 3000),
("act_runner", 8088),
("brencher-state-proxy", 80),
("docker_socket_proxy", 2375),
("telegram-mcp-server", 8000),
("gmail-mcp-server", 8000),
("voice-gateway", 8765),
("hermes-proxy", 8645),
]
@dataclass(frozen=True)
class Endpoint:
name: str
port: int
proxy: Optional[Tuple[str, int]] = None
tls: bool = False
NETWORK_MAP: Dict[str, List[Service]] = {
SWARM_NETWORKS = (
"hermes_infra_hermes_proxy",
"hermes_infra_squid_egress",
"hermes_infra_admin",
"hermes_infra_default",
"hermes_hermes",
"gitea_gitea",
"gmail_mcp_default",
"immich_default",
"torrserv_proxy_default",
"docker_socket_proxy_hermes_proxy",
"connectivity_tester_default",
"brencher_brencher-net",
"brencher2_brencher-net",
)
SQUID_PROXY: Tuple[str, int] = ("squid", 3128)
NETWORK_MAP: Dict[str, List[Endpoint]] = {
"hermes_infra_hermes_proxy": [
("hermes-gateway", 8000),
("hermes-agent", 8000),
("hermes-ui", 8087),
("squid", 3128),
("gitea-server", 3000),
("act_runner", 8088),
("brencher-state-proxy", 80),
("docker_socket_proxy", 2375),
("telegram-mcp-server", 8000),
("gmail-mcp-server", 8000),
("voice-gateway", 8765),
("hermes-proxy", 8645),
Endpoint("hermes-gateway", 8000),
Endpoint("hermes-ui", 8087),
Endpoint("squid", 3128),
Endpoint("docker_socket_proxy", 2375),
Endpoint("brencher-state-proxy", 80),
],
"hermes_infra_squid_egress": [
Endpoint("squid", 3128),
],
"hermes_infra_admin": [
Endpoint("sarg", 80),
],
"hermes_infra_default": [
Endpoint("hermes-ui", 8087),
],
"hermes_hermes": [
Endpoint("hermes-agent", 8000),
Endpoint("hermes-gateway", 8000),
],
"gitea_gitea": [
("gitea-server", 3000),
("act_runner", 8088),
("cache_server", 37051),
Endpoint("gitea-server", 3000),
Endpoint("act_runner", 8088),
Endpoint("cache_server", 37051),
],
"gmail_mcp_default": [
Endpoint("gmail-mcp-server", 8000),
Endpoint("telegram-mcp-server", 8000),
],
"immich_default": [
Endpoint("immich-server", 2283),
Endpoint("immich-database", 5432),
Endpoint("immich-redis", 6379),
Endpoint("immich-machine-learning", 8501),
],
"torrserv_proxy_default": [
Endpoint("fileserver", 8000),
Endpoint("reverseproxy", 8080),
],
"docker_socket_proxy_hermes_proxy": [
Endpoint("hermes-gateway", 8000),
],
"connectivity_tester_default": [
Endpoint("connectivity-tester", 8765),
],
"brencher_brencher-net": [
Endpoint("brencher-backend", 5000),
],
"brencher2_brencher-net": [
Endpoint("brencher-backend", 5000),
],
}
PUBLIC_SERVICES: List[Endpoint] = [
Endpoint("gitea.rudolf.keenetic.link", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("gitea.rudolf.keenetic.link", 80, proxy=SQUID_PROXY),
Endpoint("registry.rudolf.keenetic.link", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("registry.rudolf.keenetic.link", 5000, proxy=SQUID_PROXY),
Endpoint("deb.debian.org", 80, proxy=SQUID_PROXY),
Endpoint("deb.debian.org", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("github.com", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("githubusercontent.com", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("telegram.org", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("t.me", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("minimax.io", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("minimax.chat", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("minimax.com", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("pypi.org", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("google.com", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("cloudflare.com", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("wikipedia.org", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("mozilla.org", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("1.1.1.1", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("8.8.8.8", 53),
Endpoint("8.8.4.4", 53),
]
GLOBAL_SERVICES: List[Endpoint] = sorted(
{ep for eps in NETWORK_MAP.values() for ep in eps} | set(PUBLIC_SERVICES),
key=lambda e: (e.name, e.port),
)
NEGATIVE: List[Endpoint] = [
Endpoint("169.254.169.254", 80),
Endpoint("metadata.google.internal", 80),
]