feat(connectivity_tester): rotate networks per-round, cross-validate map

Self-attach/detach to overlay networks via docker socket. At any moment the
container is on exactly one network; per-network list of expected services is
loaded from network_map.NETWORK_MAP. After checks run cross-validation:

- service reachable but not in map  -> ERROR 'LEAK-UNEXPECTED'
- service in map but unreachable    -> ERROR 'MISSING-EXPECTED'
- global service not in any map     -> WARNING 'GLOBAL-UNCOVERED'

Compose uses /var/run/docker.sock bind mount and removes overlay-network
references; EXTRA_ENDPOINTS / HTTP_PROXY / HTTPS_PROXY / NO_PROXY are gone.
endpoints.py removed (its data moved into network_map.py).
This commit is contained in:
2026-09-15 09:23:55 +00:00
parent e94ea37e85
commit d5b1a23bcb
5 changed files with 204 additions and 121 deletions
+2
View File
@@ -3,6 +3,8 @@ FROM python:3.12-slim
WORKDIR /app
COPY . /app/
RUN pip install --no-cache-dir docker==7.1.0
ENV PYTHONUNBUFFERED=1 \
PERIOD_SECONDS=600 \
CHECK_TIMEOUT_S=3
+159 -58
View File
@@ -1,11 +1,10 @@
"""Connectivity tester: verifies internal/external endpoints are reachable
and negative-list endpoints are not. Logs structured output, alerts on
unexpected failures.
"""Connectivity tester.
ENV:
EXTRA_ENDPOINTS - comma-separated host:port pairs, ad-hoc checks
PERIOD_SECONDS - sleep between full check rounds (default 600)
CHECK_TIMEOUT_S - per-endpoint timeout in seconds (default 3)
For every network in network_map.NETWORK_MAP the container attaches itself,
runs TCP checks against the listed services, detaches, and proceeds to the
next network. Cross-validates GLOBAL_SERVICES against each network map: logs
errors when reachable services are absent from the map or when expected
services are unreachable.
"""
from __future__ import annotations
@@ -15,30 +14,86 @@ import os
import socket
import sys
import time
from typing import Iterable, List, Tuple
from typing import Iterable, List, Set, Tuple
from endpoints import EXPECTED_EXTERNAL, EXPECTED_INTERNAL, NEGATIVE, Endpoint
import docker
from docker.models.containers import Container
from docker.models.networks import Network
from network_map import GLOBAL_SERVICES, NETWORK_MAP, Service
LOG = logging.getLogger("connectivity-tester")
def _parse_extra(raw: str) -> List[Endpoint]:
out: List[Endpoint] = []
for token in raw.split(","):
token = token.strip()
if not token:
def _socket_path() -> str:
return os.environ.get("DOCKER_SOCKET", "/var/run/docker.sock")
def _self_container(client: docker.DockerClient) -> Container:
hostname = socket.gethostname()
for c in client.containers.list():
if (c.attrs.get("Config", {}).get("Hostname") == hostname
or c.name.lstrip("/") == hostname):
return c
candidates = [c for c in client.containers.list()
if hostname[:12] in c.id]
if len(candidates) == 1:
return candidates[0]
raise RuntimeError(f"self container not found for hostname={hostname}")
def _filtered_networks(client: docker.DockerClient) -> List[Network]:
keep_prefixes = (
"hermes_infra_", "gitea_", "gmail_mcp_", "telegram_",
"hermes_", "brencher", "registry_", "immich_",
"torrserv_proxy_", "photosHelper_", "nginx_",
"docker_socket_proxy_",
)
out = []
for n in client.networks.list():
if n.attrs.get("Scope") != "swarm":
continue
if ":" not in token:
name = n.name
if name in ("ingress",):
continue
host, port_s = token.rsplit(":", 1)
try:
port = int(port_s)
except ValueError:
continue
out.append((host, port, "extra"))
if any(name.startswith(p) for p in keep_prefixes):
out.append(n)
return out
def _build_map_from_running(networks: Iterable[Network]) -> dict:
"""Reverse-engineer NAME -> swarm network name from running container endpoints."""
out = {}
for net in networks:
endpoint_count = len(net.attrs.get("Containers") or {})
out[net.name] = {"endpoints": endpoint_count, "id": net.id}
return out
def _attach(client: docker.DockerClient, container: Container,
network: Network) -> None:
if network.name in (container.attrs.get("NetworkSettings", {})
.get("Networks", {}).keys()):
return
try:
network.connect(container, aliases=["connectivity-tester",
socket.gethostname()])
except Exception as exc:
if "already exists" in str(exc).lower():
return
raise
def _detach(client: docker.DockerClient, container: Container,
network: Network) -> None:
try:
network.disconnect(container, force=True)
except Exception as exc:
if "not found" in str(exc).lower():
return
raise
async def _tcp_check(host: str, port: int, timeout: float) -> Tuple[bool, str]:
try:
fut = asyncio.open_connection(host, port)
@@ -55,45 +110,60 @@ async def _tcp_check(host: str, port: int, timeout: float) -> Tuple[bool, str]:
return False, f"{type(exc).__name__}: {exc}"
async def _check_positive(endpoints: Iterable[Endpoint], timeout: float) -> int:
failures = 0
for host, port, label in endpoints:
ok, detail = await _tcp_check(host, port, timeout)
if ok:
LOG.info("PASS %s:%d (%s) %s", host, port, label, detail)
else:
LOG.error("FAIL %s:%d (%s) %s", host, port, label, detail)
failures += 1
return failures
async def _check_one(host: str, port: int, label: str,
timeout: float) -> bool:
ok, detail = await _tcp_check(host, port, timeout)
if ok:
LOG.info("PASS %s:%d (%s) %s", host, port, label, detail)
else:
LOG.error("FAIL %s:%d (%s) %s", host, port, label, detail)
return ok
async def _check_negative(endpoints: Iterable[Endpoint], timeout: float) -> int:
leaks = 0
for host, port, label in endpoints:
ok, detail = await _tcp_check(host, port, timeout)
if ok:
LOG.error(
"LEAK %s:%d (%s) reachable but must NOT be: %s",
host, port, label, detail,
)
leaks += 1
else:
LOG.info("OK-NEG %s:%d (%s) blocked: %s", host, port, label, detail)
return leaks
async def _check_network(network: Network, services: List[Service],
timeout: float) -> Tuple[Set[Service], Set[Service]]:
reachable: Set[Service] = set()
unreachable: Set[Service] = set()
for host, port in services:
ok = await _check_one(host, port, network.name, timeout)
(reachable if ok else unreachable).add((host, port))
return reachable, unreachable
async def _round(timeout: float) -> int:
extras = _parse_extra(os.environ.get("EXTRA_ENDPOINTS", ""))
LOG.info("--- round begin, %d internal + %d external + %d extra + %d negative ---",
len(EXPECTED_INTERNAL), len(EXPECTED_EXTERNAL), len(extras), len(NEGATIVE))
failures = 0
failures += await _check_positive(EXPECTED_INTERNAL, timeout)
failures += await _check_positive(EXPECTED_EXTERNAL, timeout)
failures += await _check_positive(extras, timeout)
leaks = await _check_negative(NEGATIVE, timeout)
failures += leaks
LOG.info("--- round end, %d failure(s) ---", failures)
return failures
def _discover_attached(client: docker.DockerClient,
container: Container) -> Set[str]:
return set((container.attrs.get("NetworkSettings", {}) or {})
.get("Networks", {}).keys())
def _cross_validate(network_name: str,
mapped: List[Service],
reachable: Set[Service],
unreachable: Set[Service],
all_global: List[Service]) -> None:
mapped_set = set(mapped)
reachable_global = reachable & set(all_global)
for svc in reachable_global - mapped_set:
LOG.error("LEAK-UNEXPECTED %s reachable in %s but not in map",
f"{svc[0]}:{svc[1]}", network_name)
for svc in mapped_set & unreachable:
LOG.error("MISSING-EXPECTED %s unreachable in %s but listed in map",
f"{svc[0]}:{svc[1]}", network_name)
for svc in mapped_set & reachable:
LOG.info("OK-MAPPED %s ok in %s", f"{svc[0]}:{svc[1]}", network_name)
for svc in mapped_set - reachable - unreachable:
LOG.warning("NOT-CHECKED %s in map for %s",
f"{svc[0]}:{svc[1]}", network_name)
def _consistency_audit(networks_names: List[str], all_global: List[Service]) -> None:
flat: Set[Service] = set()
for n in networks_names:
flat.update(NETWORK_MAP.get(n, []))
leftover = set(all_global) - flat
if leftover:
LOG.warning("GLOBAL-UNCOVERED %s not present in any map",
sorted(f"{s[0]}:{s[1]}" for s in leftover))
async def main() -> int:
@@ -104,13 +174,44 @@ async def main() -> int:
)
period = int(os.environ.get("PERIOD_SECONDS", "600"))
timeout = float(os.environ.get("CHECK_TIMEOUT_S", "3"))
LOG.info("starting; period=%ds, check_timeout=%.1fs", period, timeout)
LOG.info("starting; period=%ds timeout=%.1fs", period, timeout)
client = docker.DockerClient(base_url="unix://" + _socket_path())
self_container = _self_container(client)
self_id_short = self_container.id[:12]
networks_docker = _filtered_networks(client)
networks_known = [n for n in networks_docker if n.name in NETWORK_MAP]
LOG.info("configured networks: %d, present in swarm: %d",
len(NETWORK_MAP), len(networks_known))
for n in networks_known:
LOG.info("plan network: %s", n.name)
_consistency_audit([n.name for n in networks_known], GLOBAL_SERVICES)
while True:
started = time.monotonic()
failures = 0
try:
await _round(timeout)
for network in networks_known:
LOG.info("--- round begin, attaching to %s ---", network.name)
_attach(client, self_container, network)
try:
reachable, unreachable = await _check_network(
network, NETWORK_MAP[network.name], timeout)
_cross_validate(
network.name,
NETWORK_MAP[network.name],
reachable,
unreachable,
GLOBAL_SERVICES,
)
finally:
_detach(client, self_container, network)
LOG.info("--- round end on %s ---", network.name)
except Exception:
LOG.exception("round crashed")
elapsed = time.monotonic() - started
sleep_for = max(0.0, period - elapsed)
await asyncio.sleep(sleep_for)
-46
View File
@@ -1,46 +0,0 @@
"""Endpoint allow-list and negative-list for connectivity-tester.
EXPECTED_INTERNAL: TCP services in overlay networks that MUST be reachable.
EXPECTED_EXTERNAL: external hosts that MUST be reachable via squid egress.
NEGATIVE: hosts that MUST NOT respond (proxy / firewall sanity check).
Each entry is (host, port, label).
"""
from __future__ import annotations
from typing import List, Tuple
Endpoint = Tuple[str, int, str]
EXPECTED_INTERNAL: List[Endpoint] = [
("squid", 3128, "egress proxy"),
("hermes-gateway", 8000, "hermes messaging gateway"),
("hermes-ui", 8087, "hermes dashboard ui"),
("docker_socket_proxy", 2375, "read-only docker socket proxy"),
("brencher-state-proxy", 80, "brencher state reverse proxy"),
("gitea-server", 3000, "gitea http"),
("act_runner", 8088, "gitea actions runner"),
("telegram-mcp-server", 8000, "telegram mcp"),
("gmail-mcp-server", 8000, "gmail mcp"),
("voice-gateway", 8765, "voice realtime ws bridge"),
("hermes-proxy", 8645, "openai-compatible http egress"),
]
EXPECTED_EXTERNAL: List[Endpoint] = [
("huggingface.co", 443, "piper voice model download"),
("registry.rudolf.keenetic.link", 5000, "local docker registry"),
("gitea.rudolf.keenetic.link", 443, "public gitea"),
("pypi.org", 443, "python package index"),
("github.com", 443, "github api"),
("google.com", 443, "egress sanity check"),
("192.168.1.1", 80, "lan gateway .1"),
("192.168.0.1", 80, "lan gateway .0"),
]
NEGATIVE: List[Endpoint] = [
("169.254.169.254", 80, "cloud metadata service"),
("metadata.google.internal", 80, "gcp metadata"),
]
+40
View File
@@ -0,0 +1,40 @@
from typing import Dict, List, Tuple
Service = Tuple[str, int]
GLOBAL_SERVICES: List[Service] = [
("hermes-gateway", 8000),
("hermes-agent", 8000),
("hermes-ui", 8087),
("squid", 3128),
("gitea-server", 3000),
("act_runner", 8088),
("brencher-state-proxy", 80),
("docker_socket_proxy", 2375),
("telegram-mcp-server", 8000),
("gmail-mcp-server", 8000),
("voice-gateway", 8765),
("hermes-proxy", 8645),
]
NETWORK_MAP: Dict[str, List[Service]] = {
"hermes_infra_hermes_proxy": [
("hermes-gateway", 8000),
("hermes-agent", 8000),
("hermes-ui", 8087),
("squid", 3128),
("gitea-server", 3000),
("act_runner", 8088),
("brencher-state-proxy", 80),
("docker_socket_proxy", 2375),
("telegram-mcp-server", 8000),
("gmail-mcp-server", 8000),
("voice-gateway", 8765),
("hermes-proxy", 8645),
],
"gitea_gitea": [
("gitea-server", 3000),
("act_runner", 8088),
("cache_server", 37051),
],
}
+3 -17
View File
@@ -6,18 +6,10 @@ services:
image: registry.rudolf.keenetic.link/connectivity_tester:${version}
hostname: connectivity-tester
user: "0:0"
networks:
- hermes_proxy
- default
environment:
PERIOD_SECONDS: "600"
CHECK_TIMEOUT_S: "3"
EXTRA_ENDPOINTS: ""
HTTP_PROXY: "http://squid:3128"
HTTPS_PROXY: "http://squid:3128"
NO_PROXY: "localhost,127.0.0.1,.local,squid,hermes-gateway,hermes-agent,hermes-proxy,brencher-state-proxy,docker_socket_proxy,10.0.0.0/8"
volumes:
- /var/run/docker.sock:/var/run/docker.sock
healthcheck:
test: ["CMD-SHELL", "python -c \"import socket; s=socket.socket(); s.settimeout(2); s.connect(('squid', 3128)); s.close()\""]
test: ["CMD-SHELL", "test -S /var/run/docker.sock"]
interval: 60s
timeout: 5s
retries: 3
@@ -34,9 +26,3 @@ services:
condition: on-failure
delay: 10s
max_attempts: 3
networks:
hermes_proxy:
external: true
default:
external: true