feat(connectivity_tester): Endpoint dataclass, proxy-aware checks, public hosts
Replace tuple-of-strings with frozen Endpoint dataclass carrying optional (proxy_host, proxy_port) tuple. Add network_map.NETWORK_MAP populated from the live swarm endpoints discovered via the docker socket, and a separate PUBLIC_SERVICES list sourced from the squid allowlist plus well-known public hosts. Endpoints with proxy are checked via HTTP CONNECT through that proxy; endpoints without are probed directly via TCP after attaching to the relevant overlay network. Cross-validation rules cover leaks and missing expectations per round.
This commit is contained in:
+155
-100
@@ -1,10 +1,12 @@
|
||||
"""Connectivity tester.
|
||||
|
||||
For every network in network_map.NETWORK_MAP the container attaches itself,
|
||||
runs TCP checks against the listed services, detaches, and proceeds to the
|
||||
next network. Cross-validates GLOBAL_SERVICES against each network map: logs
|
||||
errors when reachable services are absent from the map or when expected
|
||||
services are unreachable.
|
||||
For every configured overlay network the container attaches itself, runs
|
||||
TCP checks against its endpoints, then detaches. Endpoints marked with
|
||||
``proxy=(host, port)`` are probed through that HTTP proxy (CONNECT). After
|
||||
each round a final loop checks PUBLIC_SERVICES. Cross-validation:
|
||||
- service reachable but not in map -> ERROR 'LEAK-UNEXPECTED'
|
||||
- service in map but unreachable -> ERROR 'MISSING-EXPECTED'
|
||||
- global service not in any map -> WARNING 'GLOBAL-UNCOVERED'
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -20,10 +22,13 @@ import docker
|
||||
from docker.models.containers import Container
|
||||
from docker.models.networks import Network
|
||||
|
||||
from network_map import GLOBAL_SERVICES, NETWORK_MAP, Service
|
||||
from network_map import Endpoint, GLOBAL_SERVICES, NEGATIVE, NETWORK_MAP, PUBLIC_SERVICES
|
||||
|
||||
LOG = logging.getLogger("connectivity-tester")
|
||||
|
||||
FALLBACK_TCP_TIMEOUT_S = 3.0
|
||||
HTTP_CONNECT_TIMEOUT_S = 5.0
|
||||
|
||||
|
||||
def _socket_path() -> str:
|
||||
return os.environ.get("DOCKER_SOCKET", "/var/run/docker.sock")
|
||||
@@ -32,55 +37,39 @@ def _socket_path() -> str:
|
||||
def _self_container(client: docker.DockerClient) -> Container:
|
||||
hostname = socket.gethostname()
|
||||
for c in client.containers.list():
|
||||
if (c.attrs.get("Config", {}).get("Hostname") == hostname
|
||||
or c.name.lstrip("/") == hostname):
|
||||
cn = c.name.lstrip("/")
|
||||
hn = c.attrs.get("Config", {}).get("Hostname", "")
|
||||
if hostname == cn or hostname == hn or hostname[:12] == c.id[:12]:
|
||||
return c
|
||||
candidates = [c for c in client.containers.list()
|
||||
if hostname[:12] in c.id]
|
||||
candidates = [c for c in client.containers.list() if hostname[:12] == c.id]
|
||||
if len(candidates) == 1:
|
||||
return candidates[0]
|
||||
raise RuntimeError(f"self container not found for hostname={hostname}")
|
||||
|
||||
|
||||
def _filtered_networks(client: docker.DockerClient) -> List[Network]:
|
||||
keep_prefixes = (
|
||||
"hermes_infra_", "gitea_", "gmail_mcp_", "telegram_",
|
||||
"hermes_", "brencher", "registry_", "immich_",
|
||||
"torrserv_proxy_", "photosHelper_", "nginx_",
|
||||
"docker_socket_proxy_",
|
||||
)
|
||||
out = []
|
||||
for n in client.networks.list():
|
||||
if n.attrs.get("Scope") != "swarm":
|
||||
continue
|
||||
name = n.name
|
||||
if name in ("ingress",):
|
||||
continue
|
||||
if any(name.startswith(p) for p in keep_prefixes):
|
||||
out.append(n)
|
||||
return out
|
||||
|
||||
|
||||
def _build_map_from_running(networks: Iterable[Network]) -> dict:
|
||||
"""Reverse-engineer NAME -> swarm network name from running container endpoints."""
|
||||
def _swarm_networks_for(client: docker.DockerClient,
|
||||
names: Iterable[str]) -> dict:
|
||||
out = {}
|
||||
for net in networks:
|
||||
endpoint_count = len(net.attrs.get("Containers") or {})
|
||||
out[net.name] = {"endpoints": endpoint_count, "id": net.id}
|
||||
by_name = {n.name: n for n in client.networks.list()}
|
||||
for name in names:
|
||||
net = by_name.get(name)
|
||||
if net is not None and net.attrs.get("Scope") == "swarm":
|
||||
out[name] = net
|
||||
return out
|
||||
|
||||
|
||||
def _attach(client: docker.DockerClient, container: Container,
|
||||
network: Network) -> None:
|
||||
if network.name in (container.attrs.get("NetworkSettings", {})
|
||||
.get("Networks", {}).keys()):
|
||||
return
|
||||
network: Network) -> bool:
|
||||
current = (container.attrs.get("NetworkSettings", {}) or {}).get("Networks", {})
|
||||
if network.name in current:
|
||||
return False
|
||||
try:
|
||||
network.connect(container, aliases=["connectivity-tester",
|
||||
socket.gethostname()])
|
||||
return True
|
||||
except Exception as exc:
|
||||
if "already exists" in str(exc).lower():
|
||||
return
|
||||
return False
|
||||
raise
|
||||
|
||||
|
||||
@@ -94,9 +83,9 @@ def _detach(client: docker.DockerClient, container: Container,
|
||||
raise
|
||||
|
||||
|
||||
async def _tcp_check(host: str, port: int, timeout: float) -> Tuple[bool, str]:
|
||||
async def _tcp_probe(host: str, port: int, timeout: float) -> Tuple[bool, str]:
|
||||
try:
|
||||
fut = asyncio.open_connection(host, port)
|
||||
fut = asyncio.open_connection(host=host, port=port)
|
||||
reader, writer = await asyncio.wait_for(fut, timeout=timeout)
|
||||
writer.close()
|
||||
try:
|
||||
@@ -110,60 +99,119 @@ async def _tcp_check(host: str, port: int, timeout: float) -> Tuple[bool, str]:
|
||||
return False, f"{type(exc).__name__}: {exc}"
|
||||
|
||||
|
||||
async def _check_one(host: str, port: int, label: str,
|
||||
timeout: float) -> bool:
|
||||
ok, detail = await _tcp_check(host, port, timeout)
|
||||
if ok:
|
||||
LOG.info("PASS %s:%d (%s) %s", host, port, label, detail)
|
||||
async def _proxy_connect(endpoint: Endpoint, timeout: float) -> Tuple[bool, str]:
|
||||
if endpoint.proxy is None:
|
||||
return await _tcp_probe(endpoint.name, endpoint.port, timeout=timeout)
|
||||
proxy_host, proxy_port = endpoint.proxy
|
||||
target = endpoint.name
|
||||
target_port = endpoint.port
|
||||
try:
|
||||
fut = asyncio.open_connection(host=proxy_host, port=proxy_port)
|
||||
reader, writer = await asyncio.wait_for(fut, timeout=timeout)
|
||||
req = (
|
||||
f"CONNECT {target}:{target_port} HTTP/1.1\r\n"
|
||||
f"Host: {target}:{target_port}\r\n\r\n"
|
||||
)
|
||||
writer.write(req.encode("ascii"))
|
||||
await writer.drain()
|
||||
status_line = await asyncio.wait_for(reader.readline(),
|
||||
timeout=timeout)
|
||||
if not status_line.startswith(b"HTTP/") or b" 200 " not in status_line:
|
||||
writer.close()
|
||||
try:
|
||||
await writer.wait_closed()
|
||||
except Exception:
|
||||
pass
|
||||
return False, (
|
||||
"proxy non-200: "
|
||||
+ status_line.strip().decode(errors="replace")
|
||||
)
|
||||
while True:
|
||||
line = await asyncio.wait_for(reader.readline(), timeout=timeout)
|
||||
if line in (b"\r\n", b"", b"\n"):
|
||||
break
|
||||
writer.close()
|
||||
try:
|
||||
await writer.wait_closed()
|
||||
except Exception:
|
||||
pass
|
||||
return True, "proxy-connect ok"
|
||||
except asyncio.TimeoutError:
|
||||
return False, "timeout"
|
||||
except OSError as exc:
|
||||
return False, f"{type(exc).__name__}: {exc}"
|
||||
|
||||
|
||||
async def _check_endpoint(endpoint: Endpoint, timeout: float) -> bool:
|
||||
if endpoint.proxy is not None:
|
||||
ok, detail = await _proxy_connect(endpoint, timeout)
|
||||
else:
|
||||
LOG.error("FAIL %s:%d (%s) %s", host, port, label, detail)
|
||||
ok, detail = await _tcp_probe(endpoint.name, endpoint.port,
|
||||
timeout=timeout)
|
||||
label = f"{endpoint.name}:{endpoint.port}"
|
||||
src = f"via {endpoint.proxy}" if endpoint.proxy else "direct"
|
||||
if ok:
|
||||
LOG.info("PASS %s %s %s", label, src, detail)
|
||||
else:
|
||||
LOG.error("FAIL %s %s %s", label, src, detail)
|
||||
return ok
|
||||
|
||||
|
||||
async def _check_network(network: Network, services: List[Service],
|
||||
timeout: float) -> Tuple[Set[Service], Set[Service]]:
|
||||
reachable: Set[Service] = set()
|
||||
unreachable: Set[Service] = set()
|
||||
for host, port in services:
|
||||
ok = await _check_one(host, port, network.name, timeout)
|
||||
(reachable if ok else unreachable).add((host, port))
|
||||
async def _check_network(network_name: str,
|
||||
endpoints: List[Endpoint],
|
||||
timeout: float) -> Tuple[Set[Endpoint], Set[Endpoint]]:
|
||||
reachable: Set[Endpoint] = set()
|
||||
unreachable: Set[Endpoint] = set()
|
||||
for ep in endpoints:
|
||||
ok = await _check_endpoint(ep, timeout)
|
||||
(reachable if ok else unreachable).add(ep)
|
||||
return reachable, unreachable
|
||||
|
||||
|
||||
def _discover_attached(client: docker.DockerClient,
|
||||
container: Container) -> Set[str]:
|
||||
return set((container.attrs.get("NetworkSettings", {}) or {})
|
||||
.get("Networks", {}).keys())
|
||||
|
||||
|
||||
def _cross_validate(network_name: str,
|
||||
mapped: List[Service],
|
||||
reachable: Set[Service],
|
||||
unreachable: Set[Service],
|
||||
all_global: List[Service]) -> None:
|
||||
mapped: List[Endpoint],
|
||||
reachable: Set[Endpoint],
|
||||
unreachable: Set[Endpoint],
|
||||
global_set: Set[Endpoint]) -> None:
|
||||
mapped_set = set(mapped)
|
||||
reachable_global = reachable & set(all_global)
|
||||
for svc in reachable_global - mapped_set:
|
||||
reachable_global = reachable & global_set
|
||||
for ep in reachable_global - mapped_set:
|
||||
LOG.error("LEAK-UNEXPECTED %s reachable in %s but not in map",
|
||||
f"{svc[0]}:{svc[1]}", network_name)
|
||||
for svc in mapped_set & unreachable:
|
||||
f"{ep.name}:{ep.port}", network_name)
|
||||
for ep in mapped_set & unreachable:
|
||||
LOG.error("MISSING-EXPECTED %s unreachable in %s but listed in map",
|
||||
f"{svc[0]}:{svc[1]}", network_name)
|
||||
for svc in mapped_set & reachable:
|
||||
LOG.info("OK-MAPPED %s ok in %s", f"{svc[0]}:{svc[1]}", network_name)
|
||||
for svc in mapped_set - reachable - unreachable:
|
||||
f"{ep.name}:{ep.port}", network_name)
|
||||
for ep in mapped_set & reachable:
|
||||
LOG.info("OK-MAPPED %s ok in %s", f"{ep.name}:{ep.port}", network_name)
|
||||
for ep in mapped_set - reachable - unreachable:
|
||||
LOG.warning("NOT-CHECKED %s in map for %s",
|
||||
f"{svc[0]}:{svc[1]}", network_name)
|
||||
f"{ep.name}:{ep.port}", network_name)
|
||||
|
||||
|
||||
def _consistency_audit(networks_names: List[str], all_global: List[Service]) -> None:
|
||||
flat: Set[Service] = set()
|
||||
for n in networks_names:
|
||||
def _consistency_audit(net_names: List[str]) -> None:
|
||||
flat: Set[Endpoint] = set()
|
||||
for n in net_names:
|
||||
flat.update(NETWORK_MAP.get(n, []))
|
||||
leftover = set(all_global) - flat
|
||||
if leftover:
|
||||
covered = flat | set(PUBLIC_SERVICES)
|
||||
all_global = set(GLOBAL_SERVICES)
|
||||
uncovered = all_global - covered
|
||||
if uncovered:
|
||||
LOG.warning("GLOBAL-UNCOVERED %s not present in any map",
|
||||
sorted(f"{s[0]}:{s[1]}" for s in leftover))
|
||||
sorted(f"{e.name}:{e.port}" for e in uncovered))
|
||||
|
||||
|
||||
async def _check_negative(endpoints: List[Endpoint],
|
||||
timeout: float) -> int:
|
||||
leaks = 0
|
||||
for ep in endpoints:
|
||||
ok = await _tcp_probe(ep.name, ep.port, timeout=timeout)
|
||||
if ok:
|
||||
LOG.error("LEAK %s:%d reachable but must NOT be",
|
||||
ep.name, ep.port)
|
||||
leaks += 1
|
||||
else:
|
||||
LOG.info("OK-NEG %s:%d blocked", ep.name, ep.port)
|
||||
return leaks
|
||||
|
||||
|
||||
async def main() -> int:
|
||||
@@ -178,37 +226,44 @@ async def main() -> int:
|
||||
|
||||
client = docker.DockerClient(base_url="unix://" + _socket_path())
|
||||
self_container = _self_container(client)
|
||||
self_id_short = self_container.id[:12]
|
||||
|
||||
networks_docker = _filtered_networks(client)
|
||||
networks_known = [n for n in networks_docker if n.name in NETWORK_MAP]
|
||||
nets = _swarm_networks_for(client, NETWORK_MAP.keys())
|
||||
planned = list(NETWORK_MAP.keys())
|
||||
LOG.info("configured networks: %d, present in swarm: %d",
|
||||
len(NETWORK_MAP), len(networks_known))
|
||||
for n in networks_known:
|
||||
LOG.info("plan network: %s", n.name)
|
||||
len(planned), len(nets))
|
||||
for n in nets:
|
||||
LOG.info("plan network: %s", n)
|
||||
|
||||
_consistency_audit([n.name for n in networks_known], GLOBAL_SERVICES)
|
||||
_consistency_audit(planned)
|
||||
|
||||
global_set: Set[Endpoint] = set(GLOBAL_SERVICES)
|
||||
|
||||
while True:
|
||||
started = time.monotonic()
|
||||
failures = 0
|
||||
try:
|
||||
for network in networks_known:
|
||||
LOG.info("--- round begin, attaching to %s ---", network.name)
|
||||
_attach(client, self_container, network)
|
||||
for net_name, net in nets.items():
|
||||
eps_in_map: List[Endpoint] = [
|
||||
e for e in NETWORK_MAP[net_name] if e.proxy is None
|
||||
]
|
||||
LOG.info("--- round begin, attaching to %s (%d endpoints) ---",
|
||||
net_name, len(eps_in_map))
|
||||
_attach(client, self_container, net)
|
||||
try:
|
||||
reachable, unreachable = await _check_network(
|
||||
network, NETWORK_MAP[network.name], timeout)
|
||||
_cross_validate(
|
||||
network.name,
|
||||
NETWORK_MAP[network.name],
|
||||
reachable,
|
||||
unreachable,
|
||||
GLOBAL_SERVICES,
|
||||
)
|
||||
net_name, eps_in_map, timeout)
|
||||
_cross_validate(net_name, eps_in_map,
|
||||
reachable, unreachable, global_set)
|
||||
finally:
|
||||
_detach(client, self_container, network)
|
||||
LOG.info("--- round end on %s ---", network.name)
|
||||
_detach(client, self_container, net)
|
||||
LOG.info("--- round end on %s ---", net_name)
|
||||
|
||||
LOG.info("--- public checks (%d endpoints) ---",
|
||||
len(PUBLIC_SERVICES))
|
||||
for ep in PUBLIC_SERVICES:
|
||||
await _check_endpoint(ep, max(timeout, HTTP_CONNECT_TIMEOUT_S))
|
||||
|
||||
LOG.info("--- negative checks (%d endpoints) ---", len(NEGATIVE))
|
||||
await _check_negative(NEGATIVE, timeout)
|
||||
except Exception:
|
||||
LOG.exception("round crashed")
|
||||
|
||||
|
||||
@@ -1,40 +1,122 @@
|
||||
from typing import Dict, List, Tuple
|
||||
from dataclasses import dataclass
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
Service = Tuple[str, int]
|
||||
|
||||
GLOBAL_SERVICES: List[Service] = [
|
||||
("hermes-gateway", 8000),
|
||||
("hermes-agent", 8000),
|
||||
("hermes-ui", 8087),
|
||||
("squid", 3128),
|
||||
("gitea-server", 3000),
|
||||
("act_runner", 8088),
|
||||
("brencher-state-proxy", 80),
|
||||
("docker_socket_proxy", 2375),
|
||||
("telegram-mcp-server", 8000),
|
||||
("gmail-mcp-server", 8000),
|
||||
("voice-gateway", 8765),
|
||||
("hermes-proxy", 8645),
|
||||
]
|
||||
@dataclass(frozen=True)
|
||||
class Endpoint:
|
||||
name: str
|
||||
port: int
|
||||
proxy: Optional[Tuple[str, int]] = None
|
||||
tls: bool = False
|
||||
|
||||
NETWORK_MAP: Dict[str, List[Service]] = {
|
||||
|
||||
SWARM_NETWORKS = (
|
||||
"hermes_infra_hermes_proxy",
|
||||
"hermes_infra_squid_egress",
|
||||
"hermes_infra_admin",
|
||||
"hermes_infra_default",
|
||||
"hermes_hermes",
|
||||
"gitea_gitea",
|
||||
"gmail_mcp_default",
|
||||
"immich_default",
|
||||
"torrserv_proxy_default",
|
||||
"docker_socket_proxy_hermes_proxy",
|
||||
"connectivity_tester_default",
|
||||
"brencher_brencher-net",
|
||||
"brencher2_brencher-net",
|
||||
)
|
||||
|
||||
|
||||
SQUID_PROXY: Tuple[str, int] = ("squid", 3128)
|
||||
|
||||
|
||||
NETWORK_MAP: Dict[str, List[Endpoint]] = {
|
||||
"hermes_infra_hermes_proxy": [
|
||||
("hermes-gateway", 8000),
|
||||
("hermes-agent", 8000),
|
||||
("hermes-ui", 8087),
|
||||
("squid", 3128),
|
||||
("gitea-server", 3000),
|
||||
("act_runner", 8088),
|
||||
("brencher-state-proxy", 80),
|
||||
("docker_socket_proxy", 2375),
|
||||
("telegram-mcp-server", 8000),
|
||||
("gmail-mcp-server", 8000),
|
||||
("voice-gateway", 8765),
|
||||
("hermes-proxy", 8645),
|
||||
Endpoint("hermes-gateway", 8000),
|
||||
Endpoint("hermes-ui", 8087),
|
||||
Endpoint("squid", 3128),
|
||||
Endpoint("docker_socket_proxy", 2375),
|
||||
Endpoint("brencher-state-proxy", 80),
|
||||
],
|
||||
"hermes_infra_squid_egress": [
|
||||
Endpoint("squid", 3128),
|
||||
],
|
||||
"hermes_infra_admin": [
|
||||
Endpoint("sarg", 80),
|
||||
],
|
||||
"hermes_infra_default": [
|
||||
Endpoint("hermes-ui", 8087),
|
||||
],
|
||||
"hermes_hermes": [
|
||||
Endpoint("hermes-agent", 8000),
|
||||
Endpoint("hermes-gateway", 8000),
|
||||
],
|
||||
"gitea_gitea": [
|
||||
("gitea-server", 3000),
|
||||
("act_runner", 8088),
|
||||
("cache_server", 37051),
|
||||
Endpoint("gitea-server", 3000),
|
||||
Endpoint("act_runner", 8088),
|
||||
Endpoint("cache_server", 37051),
|
||||
],
|
||||
"gmail_mcp_default": [
|
||||
Endpoint("gmail-mcp-server", 8000),
|
||||
Endpoint("telegram-mcp-server", 8000),
|
||||
],
|
||||
"immich_default": [
|
||||
Endpoint("immich-server", 2283),
|
||||
Endpoint("immich-database", 5432),
|
||||
Endpoint("immich-redis", 6379),
|
||||
Endpoint("immich-machine-learning", 8501),
|
||||
],
|
||||
"torrserv_proxy_default": [
|
||||
Endpoint("fileserver", 8000),
|
||||
Endpoint("reverseproxy", 8080),
|
||||
],
|
||||
"docker_socket_proxy_hermes_proxy": [
|
||||
Endpoint("hermes-gateway", 8000),
|
||||
],
|
||||
"connectivity_tester_default": [
|
||||
Endpoint("connectivity-tester", 8765),
|
||||
],
|
||||
"brencher_brencher-net": [
|
||||
Endpoint("brencher-backend", 5000),
|
||||
],
|
||||
"brencher2_brencher-net": [
|
||||
Endpoint("brencher-backend", 5000),
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
PUBLIC_SERVICES: List[Endpoint] = [
|
||||
Endpoint("gitea.rudolf.keenetic.link", 443, proxy=SQUID_PROXY, tls=True),
|
||||
Endpoint("gitea.rudolf.keenetic.link", 80, proxy=SQUID_PROXY),
|
||||
Endpoint("registry.rudolf.keenetic.link", 443, proxy=SQUID_PROXY, tls=True),
|
||||
Endpoint("registry.rudolf.keenetic.link", 5000, proxy=SQUID_PROXY),
|
||||
Endpoint("deb.debian.org", 80, proxy=SQUID_PROXY),
|
||||
Endpoint("deb.debian.org", 443, proxy=SQUID_PROXY, tls=True),
|
||||
Endpoint("github.com", 443, proxy=SQUID_PROXY, tls=True),
|
||||
Endpoint("githubusercontent.com", 443, proxy=SQUID_PROXY, tls=True),
|
||||
Endpoint("telegram.org", 443, proxy=SQUID_PROXY, tls=True),
|
||||
Endpoint("t.me", 443, proxy=SQUID_PROXY, tls=True),
|
||||
Endpoint("minimax.io", 443, proxy=SQUID_PROXY, tls=True),
|
||||
Endpoint("minimax.chat", 443, proxy=SQUID_PROXY, tls=True),
|
||||
Endpoint("minimax.com", 443, proxy=SQUID_PROXY, tls=True),
|
||||
Endpoint("pypi.org", 443, proxy=SQUID_PROXY, tls=True),
|
||||
Endpoint("google.com", 443, proxy=SQUID_PROXY, tls=True),
|
||||
Endpoint("cloudflare.com", 443, proxy=SQUID_PROXY, tls=True),
|
||||
Endpoint("wikipedia.org", 443, proxy=SQUID_PROXY, tls=True),
|
||||
Endpoint("mozilla.org", 443, proxy=SQUID_PROXY, tls=True),
|
||||
Endpoint("1.1.1.1", 443, proxy=SQUID_PROXY, tls=True),
|
||||
Endpoint("8.8.8.8", 53),
|
||||
Endpoint("8.8.4.4", 53),
|
||||
]
|
||||
|
||||
|
||||
GLOBAL_SERVICES: List[Endpoint] = sorted(
|
||||
{ep for eps in NETWORK_MAP.values() for ep in eps} | set(PUBLIC_SERVICES),
|
||||
key=lambda e: (e.name, e.port),
|
||||
)
|
||||
|
||||
|
||||
NEGATIVE: List[Endpoint] = [
|
||||
Endpoint("169.254.169.254", 80),
|
||||
Endpoint("metadata.google.internal", 80),
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user