refactor: move connectivity tester to dedicated repository

This commit is contained in:
2026-10-08 20:02:27 +00:00
parent db6a79e1d0
commit 248889bb6b
6 changed files with 0 additions and 638 deletions
-12
View File
@@ -1,12 +0,0 @@
FROM python:3.12-alpine
WORKDIR /app
COPY . /app/
RUN pip install --no-cache-dir docker==7.1.0
ENV PYTHONUNBUFFERED=1 \
PERIOD_SECONDS=600 \
CHECK_TIMEOUT_S=3
CMD ["python", "-m", "app"]
-360
View File
@@ -1,360 +0,0 @@
"""Connectivity tester.
For every configured overlay network the container attaches itself, runs
TCP checks against its endpoints, then detaches. Endpoints marked with
``proxy=(host, port)`` are probed through that HTTP proxy (CONNECT). After
each round a final loop checks PUBLIC_SERVICES. Cross-validation:
- service reachable but not in map -> ERROR 'LEAK-UNEXPECTED'
- service in map but unreachable -> ERROR 'MISSING-EXPECTED'
- global service not in any map -> WARNING 'GLOBAL-UNCOVERED'
"""
from __future__ import annotations
import asyncio
import logging
import os
import ssl
import socket
import sys
import time
from pathlib import Path
from typing import Iterable, List, Set, Tuple
import docker
from docker.models.containers import Container
from docker.models.networks import Network
from network_map import Endpoint, GLOBAL_SERVICES, NEGATIVE, NETWORK_MAP, PUBLIC_SERVICES
from host_checks import build_host_checks, parse_peers, tcp_probe, udp_probe
LOG = logging.getLogger("connectivity-tester")
FALLBACK_TCP_TIMEOUT_S = 3.0
HTTP_CONNECT_TIMEOUT_S = 5.0
HOST_TCP_PORTS = (2377, 7946, 2049)
HOST_UDP_PORTS = (7946, 4789)
DIRECT_INTERNET_HOST = "api.telegram.org"
DIRECT_INTERNET_PORT = 443
NFS_PROBE_PATH = os.environ.get("NFS_PROBE_PATH", "/mnt/hermes-nfs-probe")
def _socket_path() -> str:
return os.environ.get("DOCKER_SOCKET", "/var/run/docker.sock")
def _self_container(client: docker.DockerClient) -> Container:
hostname = socket.gethostname()
for c in client.containers.list():
cn = c.name.lstrip("/")
hn = c.attrs.get("Config", {}).get("Hostname", "")
if hostname == cn or hostname == hn or hostname[:12] == c.id[:12]:
return c
candidates = [c for c in client.containers.list() if hostname[:12] == c.id]
if len(candidates) == 1:
return candidates[0]
raise RuntimeError(f"self container not found for hostname={hostname}")
def _swarm_networks_for(client: docker.DockerClient,
names: Iterable[str]) -> dict:
out = {}
by_name = {n.name: n for n in client.networks.list()}
for name in names:
net = by_name.get(name)
if net is not None and net.attrs.get("Scope") == "swarm":
out[name] = net
return out
def _attach(client: docker.DockerClient, container: Container,
network: Network) -> bool:
current = (container.attrs.get("NetworkSettings", {}) or {}).get("Networks", {})
if network.name in current:
return False
try:
network.connect(container, aliases=["connectivity-tester",
socket.gethostname()])
return True
except Exception as exc:
if "already exists" in str(exc).lower():
return False
raise
def _detach(client: docker.DockerClient, container: Container,
network: Network) -> None:
try:
network.disconnect(container, force=True)
except Exception as exc:
if "not found" in str(exc).lower():
return
raise
async def _tcp_probe(host: str, port: int, timeout: float) -> Tuple[bool, str]:
try:
fut = asyncio.open_connection(host=host, port=port)
reader, writer = await asyncio.wait_for(fut, timeout=timeout)
writer.close()
try:
await writer.wait_closed()
except Exception:
pass
return True, "tcp ok"
except asyncio.TimeoutError:
return False, "timeout"
except OSError as exc:
return False, f"{type(exc).__name__}: {exc}"
async def _tls_probe(host: str, port: int, timeout: float) -> Tuple[bool, str]:
context = ssl.create_default_context()
try:
_, writer = await asyncio.wait_for(
asyncio.open_connection(host, port, ssl=context, server_hostname=host),
timeout=timeout,
)
writer.close()
try:
await writer.wait_closed()
except Exception:
pass
return True, "tls handshake ok"
except asyncio.TimeoutError:
return False, "timeout"
except (OSError, ssl.SSLError) as exc:
return False, f"{type(exc).__name__}: {exc}"
async def _proxy_connect(endpoint: Endpoint, timeout: float) -> Tuple[bool, str]:
if endpoint.proxy is None:
return await _tcp_probe(endpoint.name, endpoint.port, timeout=timeout)
proxy_host, proxy_port = endpoint.proxy
target = endpoint.name
target_port = endpoint.port
try:
fut = asyncio.open_connection(host=proxy_host, port=proxy_port)
reader, writer = await asyncio.wait_for(fut, timeout=timeout)
req = (
f"CONNECT {target}:{target_port} HTTP/1.1\r\n"
f"Host: {target}:{target_port}\r\n\r\n"
)
writer.write(req.encode("ascii"))
await writer.drain()
status_line = await asyncio.wait_for(reader.readline(),
timeout=timeout)
if not status_line.startswith(b"HTTP/") or b" 200 " not in status_line:
writer.close()
try:
await writer.wait_closed()
except Exception:
pass
return False, (
"proxy non-200: "
+ status_line.strip().decode(errors="replace")
)
while True:
line = await asyncio.wait_for(reader.readline(), timeout=timeout)
if line in (b"\r\n", b"", b"\n"):
break
writer.close()
try:
await writer.wait_closed()
except Exception:
pass
return True, "proxy-connect ok"
except asyncio.TimeoutError:
return False, "timeout"
except OSError as exc:
return False, f"{type(exc).__name__}: {exc}"
async def _check_endpoint(endpoint: Endpoint, timeout: float) -> bool:
if endpoint.proxy is not None:
ok, detail = await _proxy_connect(endpoint, timeout)
else:
ok, detail = await _tcp_probe(endpoint.name, endpoint.port,
timeout=timeout)
label = f"{endpoint.name}:{endpoint.port}"
src = f"via {endpoint.proxy}" if endpoint.proxy else "direct"
if ok:
LOG.info("PASS %s %s %s", label, src, detail)
else:
LOG.error("FAIL %s %s %s", label, src, detail)
return ok
async def _check_network(network_name: str,
endpoints: List[Endpoint],
timeout: float) -> Tuple[Set[Endpoint], Set[Endpoint]]:
reachable: Set[Endpoint] = set()
unreachable: Set[Endpoint] = set()
for ep in endpoints:
ok = await _check_endpoint(ep, timeout)
(reachable if ok else unreachable).add(ep)
return reachable, unreachable
def _cross_validate(network_name: str,
mapped: List[Endpoint],
reachable: Set[Endpoint],
unreachable: Set[Endpoint],
global_set: Set[Endpoint]) -> None:
mapped_set = set(mapped)
reachable_global = reachable & global_set
for ep in reachable_global - mapped_set:
LOG.error("LEAK-UNEXPECTED %s reachable in %s but not in map",
f"{ep.name}:{ep.port}", network_name)
for ep in mapped_set & unreachable:
LOG.error("MISSING-EXPECTED %s unreachable in %s but listed in map",
f"{ep.name}:{ep.port}", network_name)
for ep in mapped_set & reachable:
LOG.info("OK-MAPPED %s ok in %s", f"{ep.name}:{ep.port}", network_name)
for ep in mapped_set - reachable - unreachable:
LOG.warning("NOT-CHECKED %s in map for %s",
f"{ep.name}:{ep.port}", network_name)
def _consistency_audit(net_names: List[str]) -> None:
flat: Set[Endpoint] = set()
for n in net_names:
flat.update(NETWORK_MAP.get(n, []))
covered = flat | set(PUBLIC_SERVICES)
all_global = set(GLOBAL_SERVICES)
uncovered = all_global - covered
if uncovered:
LOG.warning("GLOBAL-UNCOVERED %s not present in any map",
sorted(f"{e.name}:{e.port}" for e in uncovered))
async def _check_negative(endpoints: List[Endpoint],
timeout: float) -> int:
leaks = 0
for ep in endpoints:
ok = await _tcp_probe(ep.name, ep.port, timeout=timeout)
if ok:
LOG.error("LEAK %s:%d reachable but must NOT be",
ep.name, ep.port)
leaks += 1
else:
LOG.info("OK-NEG %s:%d blocked", ep.name, ep.port)
return leaks
async def _check_host_connectivity(
peers: List[Tuple[str, str]], timeout: float
) -> None:
checks = build_host_checks(peers, HOST_TCP_PORTS, HOST_UDP_PORTS)
LOG.info("--- host connectivity checks (%d probes) ---", len(checks))
for name, address, protocol, port in checks:
if protocol == "tcp":
ok, detail = await tcp_probe(address, port, timeout)
else:
ok, detail = await udp_probe(address, port, timeout)
label = f"{name} {address}:{port}/{protocol}"
if ok:
LOG.info("PASS-HOST %s %s", label, detail)
else:
LOG.error("FAIL-HOST %s %s", label, detail)
ok, detail = await _tls_probe(
DIRECT_INTERNET_HOST,
DIRECT_INTERNET_PORT,
max(timeout, HTTP_CONNECT_TIMEOUT_S),
)
label = f"{DIRECT_INTERNET_HOST}:{DIRECT_INTERNET_PORT}"
if ok:
LOG.info("PASS-HOST %s direct %s", label, detail)
else:
LOG.error("FAIL-HOST %s direct %s", label, detail)
def _check_nfs() -> bool:
path = Path(NFS_PROBE_PATH)
probe = path / (
f".connectivity-tester-{socket.gethostname()}-"
f"{os.getpid()}-{time.time_ns()}"
)
payload = f"{socket.gethostname()}:{time.time_ns()}\n"
started = time.monotonic()
try:
probe.write_text(payload, encoding="utf-8")
if probe.read_text(encoding="utf-8") != payload:
raise OSError("read-back mismatch")
LOG.info("PASS-HOST nfs %s read/write latency=%.3fs",
NFS_PROBE_PATH, time.monotonic() - started)
return True
except OSError as exc:
LOG.error("FAIL-HOST nfs %s %s", NFS_PROBE_PATH, exc)
return False
finally:
try:
probe.unlink()
except FileNotFoundError:
pass
async def main() -> int:
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(levelname)s %(message)s",
stream=sys.stdout,
)
period = int(os.environ.get("PERIOD_SECONDS", "600"))
timeout = float(os.environ.get("CHECK_TIMEOUT_S", "3"))
peers = parse_peers(os.environ.get("SWARM_PEERS", ""))
LOG.info("starting; period=%ds timeout=%.1fs", period, timeout)
LOG.info("configured swarm peers: %d", len(peers))
client = docker.DockerClient(base_url="unix://" + _socket_path())
self_container = _self_container(client)
nets = _swarm_networks_for(client, NETWORK_MAP.keys())
planned = list(NETWORK_MAP.keys())
LOG.info("configured networks: %d, present in swarm: %d",
len(planned), len(nets))
for n in nets:
LOG.info("plan network: %s", n)
_consistency_audit(planned)
global_set: Set[Endpoint] = set(GLOBAL_SERVICES)
while True:
started = time.monotonic()
try:
for net_name, net in nets.items():
eps_in_map: List[Endpoint] = [
e for e in NETWORK_MAP[net_name] if e.proxy is None
]
LOG.info("--- round begin, attaching to %s (%d endpoints) ---",
net_name, len(eps_in_map))
_attach(client, self_container, net)
try:
reachable, unreachable = await _check_network(
net_name, eps_in_map, timeout)
_cross_validate(net_name, eps_in_map,
reachable, unreachable, global_set)
finally:
_detach(client, self_container, net)
LOG.info("--- round end on %s ---", net_name)
LOG.info("--- public checks (%d endpoints) ---",
len(PUBLIC_SERVICES))
for ep in PUBLIC_SERVICES:
await _check_endpoint(ep, max(timeout, HTTP_CONNECT_TIMEOUT_S))
LOG.info("--- negative checks (%d endpoints) ---", len(NEGATIVE))
await _check_negative(NEGATIVE, timeout)
await _check_host_connectivity(peers, timeout)
_check_nfs()
except Exception:
LOG.exception("round crashed")
elapsed = time.monotonic() - started
sleep_for = max(0.0, period - elapsed)
await asyncio.sleep(sleep_for)
if __name__ == "__main__":
asyncio.run(main())
-71
View File
@@ -1,71 +0,0 @@
from __future__ import annotations
import asyncio
import socket
from dataclasses import dataclass
from typing import Iterable, List, Tuple
@dataclass(frozen=True)
class HostCheck:
name: str
address: str
protocol: str
port: int
def parse_peers(value: str) -> List[Tuple[str, str]]:
peers = []
for item in value.split(","):
item = item.strip()
if "=" not in item:
continue
name, address = (part.strip() for part in item.split("=", 1))
if name and address:
peers.append((name, address))
return peers
def build_host_checks(
peers: Iterable[Tuple[str, str]],
tcp_ports: Iterable[int],
udp_ports: Iterable[int],
) -> List[Tuple[str, str, str, int]]:
checks = []
for name, address in peers:
checks.extend((name, address, "tcp", port) for port in tcp_ports)
checks.extend((name, address, "udp", port) for port in udp_ports)
return checks
async def tcp_probe(address: str, port: int, timeout: float) -> Tuple[bool, str]:
try:
_, writer = await asyncio.wait_for(
asyncio.open_connection(address, port), timeout=timeout
)
writer.close()
try:
await writer.wait_closed()
except Exception:
pass
return True, "tcp connect ok"
except asyncio.TimeoutError:
return False, "timeout"
except OSError as exc:
return False, f"{type(exc).__name__}: {exc}"
async def udp_probe(address: str, port: int, timeout: float) -> Tuple[bool, str]:
loop = asyncio.get_running_loop()
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.setblocking(False)
try:
await asyncio.wait_for(loop.sock_connect(sock, (address, port)), timeout)
await asyncio.wait_for(loop.sock_sendall(sock, b"connectivity-tester\n"), timeout)
return True, "udp datagram sent; no handshake available"
except asyncio.TimeoutError:
return False, "timeout"
except OSError as exc:
return False, f"{type(exc).__name__}: {exc}"
finally:
sock.close()
-118
View File
@@ -1,118 +0,0 @@
from dataclasses import dataclass
from typing import Dict, List, Optional, Tuple
@dataclass(frozen=True)
class Endpoint:
name: str
port: int
proxy: Optional[Tuple[str, int]] = None
tls: bool = False
SWARM_NETWORKS = (
"hermes_infra_hermes_proxy",
"hermes_infra_squid_egress",
"hermes_infra_admin",
"hermes_infra_default",
"hermes_hermes",
"gitea_gitea",
"gmail_mcp_default",
"immich_default",
"torrserv_proxy_default",
"docker_socket_proxy_hermes_proxy",
"brencher_brencher-net",
"brencher2_brencher-net",
)
SQUID_PROXY: Tuple[str, int] = ("squid", 3128)
NETWORK_MAP: Dict[str, List[Endpoint]] = {
"hermes_infra_hermes_proxy": [
Endpoint("hermes-gateway", 8000),
Endpoint("hermes-ui", 8087),
Endpoint("squid", 3128),
Endpoint("docker_socket_proxy", 2375),
Endpoint("brencher-state-proxy", 80),
],
"hermes_infra_squid_egress": [
Endpoint("squid", 3128),
],
"hermes_infra_admin": [
Endpoint("sarg", 80),
],
"hermes_infra_default": [
Endpoint("hermes-ui", 8087),
],
"hermes_hermes": [
Endpoint("hermes-agent", 8000),
Endpoint("hermes-gateway", 8000),
],
"gitea_gitea": [
Endpoint("gitea-server", 3000),
Endpoint("act_runner", 8088),
Endpoint("cache_server", 37051),
],
"gmail_mcp_default": [
Endpoint("gmail-mcp-server", 8000),
Endpoint("telegram-mcp-server", 8000),
],
"immich_default": [
Endpoint("immich-server", 2283),
Endpoint("immich-database", 5432),
Endpoint("immich-redis", 6379),
Endpoint("immich-machine-learning", 8501),
],
"torrserv_proxy_default": [
Endpoint("fileserver", 8000),
Endpoint("reverseproxy", 8080),
],
"docker_socket_proxy_hermes_proxy": [
Endpoint("hermes-gateway", 8000),
],
"brencher_brencher-net": [
Endpoint("brencher-backend", 5000),
],
"brencher2_brencher-net": [
Endpoint("brencher-backend", 5000),
],
}
PUBLIC_SERVICES: List[Endpoint] = [
Endpoint("gitea.rudolf.keenetic.link", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("gitea.rudolf.keenetic.link", 80, proxy=SQUID_PROXY),
Endpoint("registry.rudolf.keenetic.link", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("registry.rudolf.keenetic.link", 5000, proxy=SQUID_PROXY),
Endpoint("deb.debian.org", 80, proxy=SQUID_PROXY),
Endpoint("deb.debian.org", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("github.com", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("githubusercontent.com", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("telegram.org", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("t.me", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("minimax.io", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("minimax.chat", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("minimax.com", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("pypi.org", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("google.com", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("cloudflare.com", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("wikipedia.org", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("mozilla.org", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("1.1.1.1", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("8.8.8.8", 53),
Endpoint("8.8.4.4", 53),
]
GLOBAL_SERVICES: List[Endpoint] = sorted(
{ep for eps in NETWORK_MAP.values() for ep in eps} | set(PUBLIC_SERVICES),
key=lambda e: (e.name, e.port),
)
NEGATIVE: List[Endpoint] = [
Endpoint("169.254.169.254", 80),
Endpoint("metadata.google.internal", 80),
]
-36
View File
@@ -1,36 +0,0 @@
services:
connectivity-tester:
image: gitea.rudolf.keenetic.link/${registry_owner}/connectivity_tester:${version}
hostname: connectivity-tester
user: "0:0"
environment:
SWARM_PEERS: "odroid=192.168.1.96,rudolf-T=100.105.201.15"
volumes:
- /var/run/docker.sock:/var/run/docker.sock
- hermes_nfs_probe:/mnt/hermes-nfs-probe
healthcheck:
test: ["CMD-SHELL", "test -S /var/run/docker.sock"]
interval: 60s
timeout: 5s
retries: 3
start_period: 30s
deploy:
mode: global
resources:
limits:
memory: 128M
pids: 64
reservations:
memory: 64M
restart_policy:
condition: on-failure
delay: 10s
max_attempts: 3
volumes:
hermes_nfs_probe:
driver: local
driver_opts:
type: nfs
o: addr=192.168.1.96,rw,nfsvers=4
device: :/root/volumes/hermes/agent
-41
View File
@@ -1,41 +0,0 @@
import unittest
from host_checks import build_host_checks, parse_peers
class HostChecksTests(unittest.TestCase):
def test_parse_peers(self):
self.assertEqual(
parse_peers("odroid=192.168.1.96,rudolf-T=100.105.201.15"),
[("odroid", "192.168.1.96"), ("rudolf-T", "100.105.201.15")],
)
def test_parse_peers_ignores_invalid_entries(self):
self.assertEqual(
parse_peers("odroid=192.168.1.96,broken,rudolf-T="),
[("odroid", "192.168.1.96")],
)
def test_build_host_checks(self):
checks = build_host_checks(
[("odroid", "192.168.1.96"), ("rudolf-T", "100.105.201.15")],
[2377, 7946],
[7946, 4789],
)
self.assertEqual(
checks,
[
("odroid", "192.168.1.96", "tcp", 2377),
("odroid", "192.168.1.96", "tcp", 7946),
("odroid", "192.168.1.96", "udp", 7946),
("odroid", "192.168.1.96", "udp", 4789),
("rudolf-T", "100.105.201.15", "tcp", 2377),
("rudolf-T", "100.105.201.15", "tcp", 7946),
("rudolf-T", "100.105.201.15", "udp", 7946),
("rudolf-T", "100.105.201.15", "udp", 4789),
],
)
if __name__ == "__main__":
unittest.main()