From 248889bb6bfac57d635bae0086332128bba47d67 Mon Sep 17 00:00:00 2001 From: brencher_bot Date: Thu, 8 Oct 2026 20:02:27 +0000 Subject: [PATCH] refactor: move connectivity tester to dedicated repository --- connectivity_tester/Dockerfile | 12 - connectivity_tester/app.py | 360 ------------------------ connectivity_tester/host_checks.py | 71 ----- connectivity_tester/network_map.py | 118 -------- connectivity_tester/stack-compose.yml | 36 --- connectivity_tester/test_host_checks.py | 41 --- 6 files changed, 638 deletions(-) delete mode 100644 connectivity_tester/Dockerfile delete mode 100644 connectivity_tester/app.py delete mode 100644 connectivity_tester/host_checks.py delete mode 100644 connectivity_tester/network_map.py delete mode 100644 connectivity_tester/stack-compose.yml delete mode 100644 connectivity_tester/test_host_checks.py diff --git a/connectivity_tester/Dockerfile b/connectivity_tester/Dockerfile deleted file mode 100644 index 49c1cb3..0000000 --- a/connectivity_tester/Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -FROM python:3.12-alpine - -WORKDIR /app -COPY . /app/ - -RUN pip install --no-cache-dir docker==7.1.0 - -ENV PYTHONUNBUFFERED=1 \ - PERIOD_SECONDS=600 \ - CHECK_TIMEOUT_S=3 - -CMD ["python", "-m", "app"] diff --git a/connectivity_tester/app.py b/connectivity_tester/app.py deleted file mode 100644 index d032d33..0000000 --- a/connectivity_tester/app.py +++ /dev/null @@ -1,360 +0,0 @@ -"""Connectivity tester. - -For every configured overlay network the container attaches itself, runs -TCP checks against its endpoints, then detaches. Endpoints marked with -``proxy=(host, port)`` are probed through that HTTP proxy (CONNECT). After -each round a final loop checks PUBLIC_SERVICES. Cross-validation: -- service reachable but not in map -> ERROR 'LEAK-UNEXPECTED' -- service in map but unreachable -> ERROR 'MISSING-EXPECTED' -- global service not in any map -> WARNING 'GLOBAL-UNCOVERED' -""" -from __future__ import annotations - -import asyncio -import logging -import os -import ssl -import socket -import sys -import time -from pathlib import Path -from typing import Iterable, List, Set, Tuple - -import docker -from docker.models.containers import Container -from docker.models.networks import Network - -from network_map import Endpoint, GLOBAL_SERVICES, NEGATIVE, NETWORK_MAP, PUBLIC_SERVICES -from host_checks import build_host_checks, parse_peers, tcp_probe, udp_probe - -LOG = logging.getLogger("connectivity-tester") - -FALLBACK_TCP_TIMEOUT_S = 3.0 -HTTP_CONNECT_TIMEOUT_S = 5.0 -HOST_TCP_PORTS = (2377, 7946, 2049) -HOST_UDP_PORTS = (7946, 4789) -DIRECT_INTERNET_HOST = "api.telegram.org" -DIRECT_INTERNET_PORT = 443 -NFS_PROBE_PATH = os.environ.get("NFS_PROBE_PATH", "/mnt/hermes-nfs-probe") - - -def _socket_path() -> str: - return os.environ.get("DOCKER_SOCKET", "/var/run/docker.sock") - - -def _self_container(client: docker.DockerClient) -> Container: - hostname = socket.gethostname() - for c in client.containers.list(): - cn = c.name.lstrip("/") - hn = c.attrs.get("Config", {}).get("Hostname", "") - if hostname == cn or hostname == hn or hostname[:12] == c.id[:12]: - return c - candidates = [c for c in client.containers.list() if hostname[:12] == c.id] - if len(candidates) == 1: - return candidates[0] - raise RuntimeError(f"self container not found for hostname={hostname}") - - -def _swarm_networks_for(client: docker.DockerClient, - names: Iterable[str]) -> dict: - out = {} - by_name = {n.name: n for n in client.networks.list()} - for name in names: - net = by_name.get(name) - if net is not None and net.attrs.get("Scope") == "swarm": - out[name] = net - return out - - -def _attach(client: docker.DockerClient, container: Container, - network: Network) -> bool: - current = (container.attrs.get("NetworkSettings", {}) or {}).get("Networks", {}) - if network.name in current: - return False - try: - network.connect(container, aliases=["connectivity-tester", - socket.gethostname()]) - return True - except Exception as exc: - if "already exists" in str(exc).lower(): - return False - raise - - -def _detach(client: docker.DockerClient, container: Container, - network: Network) -> None: - try: - network.disconnect(container, force=True) - except Exception as exc: - if "not found" in str(exc).lower(): - return - raise - - -async def _tcp_probe(host: str, port: int, timeout: float) -> Tuple[bool, str]: - try: - fut = asyncio.open_connection(host=host, port=port) - reader, writer = await asyncio.wait_for(fut, timeout=timeout) - writer.close() - try: - await writer.wait_closed() - except Exception: - pass - return True, "tcp ok" - except asyncio.TimeoutError: - return False, "timeout" - except OSError as exc: - return False, f"{type(exc).__name__}: {exc}" - - -async def _tls_probe(host: str, port: int, timeout: float) -> Tuple[bool, str]: - context = ssl.create_default_context() - try: - _, writer = await asyncio.wait_for( - asyncio.open_connection(host, port, ssl=context, server_hostname=host), - timeout=timeout, - ) - writer.close() - try: - await writer.wait_closed() - except Exception: - pass - return True, "tls handshake ok" - except asyncio.TimeoutError: - return False, "timeout" - except (OSError, ssl.SSLError) as exc: - return False, f"{type(exc).__name__}: {exc}" - - -async def _proxy_connect(endpoint: Endpoint, timeout: float) -> Tuple[bool, str]: - if endpoint.proxy is None: - return await _tcp_probe(endpoint.name, endpoint.port, timeout=timeout) - proxy_host, proxy_port = endpoint.proxy - target = endpoint.name - target_port = endpoint.port - try: - fut = asyncio.open_connection(host=proxy_host, port=proxy_port) - reader, writer = await asyncio.wait_for(fut, timeout=timeout) - req = ( - f"CONNECT {target}:{target_port} HTTP/1.1\r\n" - f"Host: {target}:{target_port}\r\n\r\n" - ) - writer.write(req.encode("ascii")) - await writer.drain() - status_line = await asyncio.wait_for(reader.readline(), - timeout=timeout) - if not status_line.startswith(b"HTTP/") or b" 200 " not in status_line: - writer.close() - try: - await writer.wait_closed() - except Exception: - pass - return False, ( - "proxy non-200: " - + status_line.strip().decode(errors="replace") - ) - while True: - line = await asyncio.wait_for(reader.readline(), timeout=timeout) - if line in (b"\r\n", b"", b"\n"): - break - writer.close() - try: - await writer.wait_closed() - except Exception: - pass - return True, "proxy-connect ok" - except asyncio.TimeoutError: - return False, "timeout" - except OSError as exc: - return False, f"{type(exc).__name__}: {exc}" - - -async def _check_endpoint(endpoint: Endpoint, timeout: float) -> bool: - if endpoint.proxy is not None: - ok, detail = await _proxy_connect(endpoint, timeout) - else: - ok, detail = await _tcp_probe(endpoint.name, endpoint.port, - timeout=timeout) - label = f"{endpoint.name}:{endpoint.port}" - src = f"via {endpoint.proxy}" if endpoint.proxy else "direct" - if ok: - LOG.info("PASS %s %s %s", label, src, detail) - else: - LOG.error("FAIL %s %s %s", label, src, detail) - return ok - - -async def _check_network(network_name: str, - endpoints: List[Endpoint], - timeout: float) -> Tuple[Set[Endpoint], Set[Endpoint]]: - reachable: Set[Endpoint] = set() - unreachable: Set[Endpoint] = set() - for ep in endpoints: - ok = await _check_endpoint(ep, timeout) - (reachable if ok else unreachable).add(ep) - return reachable, unreachable - - -def _cross_validate(network_name: str, - mapped: List[Endpoint], - reachable: Set[Endpoint], - unreachable: Set[Endpoint], - global_set: Set[Endpoint]) -> None: - mapped_set = set(mapped) - reachable_global = reachable & global_set - for ep in reachable_global - mapped_set: - LOG.error("LEAK-UNEXPECTED %s reachable in %s but not in map", - f"{ep.name}:{ep.port}", network_name) - for ep in mapped_set & unreachable: - LOG.error("MISSING-EXPECTED %s unreachable in %s but listed in map", - f"{ep.name}:{ep.port}", network_name) - for ep in mapped_set & reachable: - LOG.info("OK-MAPPED %s ok in %s", f"{ep.name}:{ep.port}", network_name) - for ep in mapped_set - reachable - unreachable: - LOG.warning("NOT-CHECKED %s in map for %s", - f"{ep.name}:{ep.port}", network_name) - - -def _consistency_audit(net_names: List[str]) -> None: - flat: Set[Endpoint] = set() - for n in net_names: - flat.update(NETWORK_MAP.get(n, [])) - covered = flat | set(PUBLIC_SERVICES) - all_global = set(GLOBAL_SERVICES) - uncovered = all_global - covered - if uncovered: - LOG.warning("GLOBAL-UNCOVERED %s not present in any map", - sorted(f"{e.name}:{e.port}" for e in uncovered)) - - -async def _check_negative(endpoints: List[Endpoint], - timeout: float) -> int: - leaks = 0 - for ep in endpoints: - ok = await _tcp_probe(ep.name, ep.port, timeout=timeout) - if ok: - LOG.error("LEAK %s:%d reachable but must NOT be", - ep.name, ep.port) - leaks += 1 - else: - LOG.info("OK-NEG %s:%d blocked", ep.name, ep.port) - return leaks - - -async def _check_host_connectivity( - peers: List[Tuple[str, str]], timeout: float -) -> None: - checks = build_host_checks(peers, HOST_TCP_PORTS, HOST_UDP_PORTS) - LOG.info("--- host connectivity checks (%d probes) ---", len(checks)) - for name, address, protocol, port in checks: - if protocol == "tcp": - ok, detail = await tcp_probe(address, port, timeout) - else: - ok, detail = await udp_probe(address, port, timeout) - label = f"{name} {address}:{port}/{protocol}" - if ok: - LOG.info("PASS-HOST %s %s", label, detail) - else: - LOG.error("FAIL-HOST %s %s", label, detail) - - ok, detail = await _tls_probe( - DIRECT_INTERNET_HOST, - DIRECT_INTERNET_PORT, - max(timeout, HTTP_CONNECT_TIMEOUT_S), - ) - label = f"{DIRECT_INTERNET_HOST}:{DIRECT_INTERNET_PORT}" - if ok: - LOG.info("PASS-HOST %s direct %s", label, detail) - else: - LOG.error("FAIL-HOST %s direct %s", label, detail) - - -def _check_nfs() -> bool: - path = Path(NFS_PROBE_PATH) - probe = path / ( - f".connectivity-tester-{socket.gethostname()}-" - f"{os.getpid()}-{time.time_ns()}" - ) - payload = f"{socket.gethostname()}:{time.time_ns()}\n" - started = time.monotonic() - try: - probe.write_text(payload, encoding="utf-8") - if probe.read_text(encoding="utf-8") != payload: - raise OSError("read-back mismatch") - LOG.info("PASS-HOST nfs %s read/write latency=%.3fs", - NFS_PROBE_PATH, time.monotonic() - started) - return True - except OSError as exc: - LOG.error("FAIL-HOST nfs %s %s", NFS_PROBE_PATH, exc) - return False - finally: - try: - probe.unlink() - except FileNotFoundError: - pass - - -async def main() -> int: - logging.basicConfig( - level=logging.INFO, - format="%(asctime)s %(levelname)s %(message)s", - stream=sys.stdout, - ) - period = int(os.environ.get("PERIOD_SECONDS", "600")) - timeout = float(os.environ.get("CHECK_TIMEOUT_S", "3")) - peers = parse_peers(os.environ.get("SWARM_PEERS", "")) - LOG.info("starting; period=%ds timeout=%.1fs", period, timeout) - LOG.info("configured swarm peers: %d", len(peers)) - - client = docker.DockerClient(base_url="unix://" + _socket_path()) - self_container = _self_container(client) - - nets = _swarm_networks_for(client, NETWORK_MAP.keys()) - planned = list(NETWORK_MAP.keys()) - LOG.info("configured networks: %d, present in swarm: %d", - len(planned), len(nets)) - for n in nets: - LOG.info("plan network: %s", n) - - _consistency_audit(planned) - - global_set: Set[Endpoint] = set(GLOBAL_SERVICES) - - while True: - started = time.monotonic() - try: - for net_name, net in nets.items(): - eps_in_map: List[Endpoint] = [ - e for e in NETWORK_MAP[net_name] if e.proxy is None - ] - LOG.info("--- round begin, attaching to %s (%d endpoints) ---", - net_name, len(eps_in_map)) - _attach(client, self_container, net) - try: - reachable, unreachable = await _check_network( - net_name, eps_in_map, timeout) - _cross_validate(net_name, eps_in_map, - reachable, unreachable, global_set) - finally: - _detach(client, self_container, net) - LOG.info("--- round end on %s ---", net_name) - - LOG.info("--- public checks (%d endpoints) ---", - len(PUBLIC_SERVICES)) - for ep in PUBLIC_SERVICES: - await _check_endpoint(ep, max(timeout, HTTP_CONNECT_TIMEOUT_S)) - - LOG.info("--- negative checks (%d endpoints) ---", len(NEGATIVE)) - await _check_negative(NEGATIVE, timeout) - await _check_host_connectivity(peers, timeout) - _check_nfs() - except Exception: - LOG.exception("round crashed") - - elapsed = time.monotonic() - started - sleep_for = max(0.0, period - elapsed) - await asyncio.sleep(sleep_for) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/connectivity_tester/host_checks.py b/connectivity_tester/host_checks.py deleted file mode 100644 index c7af720..0000000 --- a/connectivity_tester/host_checks.py +++ /dev/null @@ -1,71 +0,0 @@ -from __future__ import annotations - -import asyncio -import socket -from dataclasses import dataclass -from typing import Iterable, List, Tuple - - -@dataclass(frozen=True) -class HostCheck: - name: str - address: str - protocol: str - port: int - - -def parse_peers(value: str) -> List[Tuple[str, str]]: - peers = [] - for item in value.split(","): - item = item.strip() - if "=" not in item: - continue - name, address = (part.strip() for part in item.split("=", 1)) - if name and address: - peers.append((name, address)) - return peers - - -def build_host_checks( - peers: Iterable[Tuple[str, str]], - tcp_ports: Iterable[int], - udp_ports: Iterable[int], -) -> List[Tuple[str, str, str, int]]: - checks = [] - for name, address in peers: - checks.extend((name, address, "tcp", port) for port in tcp_ports) - checks.extend((name, address, "udp", port) for port in udp_ports) - return checks - - -async def tcp_probe(address: str, port: int, timeout: float) -> Tuple[bool, str]: - try: - _, writer = await asyncio.wait_for( - asyncio.open_connection(address, port), timeout=timeout - ) - writer.close() - try: - await writer.wait_closed() - except Exception: - pass - return True, "tcp connect ok" - except asyncio.TimeoutError: - return False, "timeout" - except OSError as exc: - return False, f"{type(exc).__name__}: {exc}" - - -async def udp_probe(address: str, port: int, timeout: float) -> Tuple[bool, str]: - loop = asyncio.get_running_loop() - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - sock.setblocking(False) - try: - await asyncio.wait_for(loop.sock_connect(sock, (address, port)), timeout) - await asyncio.wait_for(loop.sock_sendall(sock, b"connectivity-tester\n"), timeout) - return True, "udp datagram sent; no handshake available" - except asyncio.TimeoutError: - return False, "timeout" - except OSError as exc: - return False, f"{type(exc).__name__}: {exc}" - finally: - sock.close() diff --git a/connectivity_tester/network_map.py b/connectivity_tester/network_map.py deleted file mode 100644 index b7aea5a..0000000 --- a/connectivity_tester/network_map.py +++ /dev/null @@ -1,118 +0,0 @@ -from dataclasses import dataclass -from typing import Dict, List, Optional, Tuple - - -@dataclass(frozen=True) -class Endpoint: - name: str - port: int - proxy: Optional[Tuple[str, int]] = None - tls: bool = False - - -SWARM_NETWORKS = ( - "hermes_infra_hermes_proxy", - "hermes_infra_squid_egress", - "hermes_infra_admin", - "hermes_infra_default", - "hermes_hermes", - "gitea_gitea", - "gmail_mcp_default", - "immich_default", - "torrserv_proxy_default", - "docker_socket_proxy_hermes_proxy", - "brencher_brencher-net", - "brencher2_brencher-net", -) - - -SQUID_PROXY: Tuple[str, int] = ("squid", 3128) - - -NETWORK_MAP: Dict[str, List[Endpoint]] = { - "hermes_infra_hermes_proxy": [ - Endpoint("hermes-gateway", 8000), - Endpoint("hermes-ui", 8087), - Endpoint("squid", 3128), - Endpoint("docker_socket_proxy", 2375), - Endpoint("brencher-state-proxy", 80), - ], - "hermes_infra_squid_egress": [ - Endpoint("squid", 3128), - ], - "hermes_infra_admin": [ - Endpoint("sarg", 80), - ], - "hermes_infra_default": [ - Endpoint("hermes-ui", 8087), - ], - "hermes_hermes": [ - Endpoint("hermes-agent", 8000), - Endpoint("hermes-gateway", 8000), - ], - "gitea_gitea": [ - Endpoint("gitea-server", 3000), - Endpoint("act_runner", 8088), - Endpoint("cache_server", 37051), - ], - "gmail_mcp_default": [ - Endpoint("gmail-mcp-server", 8000), - Endpoint("telegram-mcp-server", 8000), - ], - "immich_default": [ - Endpoint("immich-server", 2283), - Endpoint("immich-database", 5432), - Endpoint("immich-redis", 6379), - Endpoint("immich-machine-learning", 8501), - ], - "torrserv_proxy_default": [ - Endpoint("fileserver", 8000), - Endpoint("reverseproxy", 8080), - ], - "docker_socket_proxy_hermes_proxy": [ - Endpoint("hermes-gateway", 8000), - ], - "brencher_brencher-net": [ - Endpoint("brencher-backend", 5000), - ], - "brencher2_brencher-net": [ - Endpoint("brencher-backend", 5000), - ], -} - - -PUBLIC_SERVICES: List[Endpoint] = [ - Endpoint("gitea.rudolf.keenetic.link", 443, proxy=SQUID_PROXY, tls=True), - Endpoint("gitea.rudolf.keenetic.link", 80, proxy=SQUID_PROXY), - Endpoint("registry.rudolf.keenetic.link", 443, proxy=SQUID_PROXY, tls=True), - Endpoint("registry.rudolf.keenetic.link", 5000, proxy=SQUID_PROXY), - Endpoint("deb.debian.org", 80, proxy=SQUID_PROXY), - Endpoint("deb.debian.org", 443, proxy=SQUID_PROXY, tls=True), - Endpoint("github.com", 443, proxy=SQUID_PROXY, tls=True), - Endpoint("githubusercontent.com", 443, proxy=SQUID_PROXY, tls=True), - Endpoint("telegram.org", 443, proxy=SQUID_PROXY, tls=True), - Endpoint("t.me", 443, proxy=SQUID_PROXY, tls=True), - Endpoint("minimax.io", 443, proxy=SQUID_PROXY, tls=True), - Endpoint("minimax.chat", 443, proxy=SQUID_PROXY, tls=True), - Endpoint("minimax.com", 443, proxy=SQUID_PROXY, tls=True), - Endpoint("pypi.org", 443, proxy=SQUID_PROXY, tls=True), - Endpoint("google.com", 443, proxy=SQUID_PROXY, tls=True), - Endpoint("cloudflare.com", 443, proxy=SQUID_PROXY, tls=True), - Endpoint("wikipedia.org", 443, proxy=SQUID_PROXY, tls=True), - Endpoint("mozilla.org", 443, proxy=SQUID_PROXY, tls=True), - Endpoint("1.1.1.1", 443, proxy=SQUID_PROXY, tls=True), - Endpoint("8.8.8.8", 53), - Endpoint("8.8.4.4", 53), -] - - -GLOBAL_SERVICES: List[Endpoint] = sorted( - {ep for eps in NETWORK_MAP.values() for ep in eps} | set(PUBLIC_SERVICES), - key=lambda e: (e.name, e.port), -) - - -NEGATIVE: List[Endpoint] = [ - Endpoint("169.254.169.254", 80), - Endpoint("metadata.google.internal", 80), -] diff --git a/connectivity_tester/stack-compose.yml b/connectivity_tester/stack-compose.yml deleted file mode 100644 index 666e0e6..0000000 --- a/connectivity_tester/stack-compose.yml +++ /dev/null @@ -1,36 +0,0 @@ -services: - connectivity-tester: - image: gitea.rudolf.keenetic.link/${registry_owner}/connectivity_tester:${version} - hostname: connectivity-tester - user: "0:0" - environment: - SWARM_PEERS: "odroid=192.168.1.96,rudolf-T=100.105.201.15" - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - hermes_nfs_probe:/mnt/hermes-nfs-probe - healthcheck: - test: ["CMD-SHELL", "test -S /var/run/docker.sock"] - interval: 60s - timeout: 5s - retries: 3 - start_period: 30s - deploy: - mode: global - resources: - limits: - memory: 128M - pids: 64 - reservations: - memory: 64M - restart_policy: - condition: on-failure - delay: 10s - max_attempts: 3 - -volumes: - hermes_nfs_probe: - driver: local - driver_opts: - type: nfs - o: addr=192.168.1.96,rw,nfsvers=4 - device: :/root/volumes/hermes/agent diff --git a/connectivity_tester/test_host_checks.py b/connectivity_tester/test_host_checks.py deleted file mode 100644 index 7018e37..0000000 --- a/connectivity_tester/test_host_checks.py +++ /dev/null @@ -1,41 +0,0 @@ -import unittest - -from host_checks import build_host_checks, parse_peers - - -class HostChecksTests(unittest.TestCase): - def test_parse_peers(self): - self.assertEqual( - parse_peers("odroid=192.168.1.96,rudolf-T=100.105.201.15"), - [("odroid", "192.168.1.96"), ("rudolf-T", "100.105.201.15")], - ) - - def test_parse_peers_ignores_invalid_entries(self): - self.assertEqual( - parse_peers("odroid=192.168.1.96,broken,rudolf-T="), - [("odroid", "192.168.1.96")], - ) - - def test_build_host_checks(self): - checks = build_host_checks( - [("odroid", "192.168.1.96"), ("rudolf-T", "100.105.201.15")], - [2377, 7946], - [7946, 4789], - ) - self.assertEqual( - checks, - [ - ("odroid", "192.168.1.96", "tcp", 2377), - ("odroid", "192.168.1.96", "tcp", 7946), - ("odroid", "192.168.1.96", "udp", 7946), - ("odroid", "192.168.1.96", "udp", 4789), - ("rudolf-T", "100.105.201.15", "tcp", 2377), - ("rudolf-T", "100.105.201.15", "tcp", 7946), - ("rudolf-T", "100.105.201.15", "udp", 7946), - ("rudolf-T", "100.105.201.15", "udp", 4789), - ], - ) - - -if __name__ == "__main__": - unittest.main()