diff --git a/connectivity_tester/Dockerfile b/connectivity_tester/Dockerfile deleted file mode 100644 index 65dc943..0000000 --- a/connectivity_tester/Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -FROM python:3.12-slim - -WORKDIR /app -COPY . /app/ - -RUN pip install --no-cache-dir docker==7.1.0 - -ENV PYTHONUNBUFFERED=1 \ - PERIOD_SECONDS=600 \ - CHECK_TIMEOUT_S=3 - -CMD ["python", "-m", "app"] diff --git a/connectivity_tester/app.py b/connectivity_tester/app.py deleted file mode 100644 index e34c6e3..0000000 --- a/connectivity_tester/app.py +++ /dev/null @@ -1,276 +0,0 @@ -"""Connectivity tester. - -For every configured overlay network the container attaches itself, runs -TCP checks against its endpoints, then detaches. Endpoints marked with -``proxy=(host, port)`` are probed through that HTTP proxy (CONNECT). After -each round a final loop checks PUBLIC_SERVICES. Cross-validation: -- service reachable but not in map -> ERROR 'LEAK-UNEXPECTED' -- service in map but unreachable -> ERROR 'MISSING-EXPECTED' -- global service not in any map -> WARNING 'GLOBAL-UNCOVERED' -""" -from __future__ import annotations - -import asyncio -import logging -import os -import socket -import sys -import time -from typing import Iterable, List, Set, Tuple - -import docker -from docker.models.containers import Container -from docker.models.networks import Network - -from network_map import Endpoint, GLOBAL_SERVICES, NEGATIVE, NETWORK_MAP, PUBLIC_SERVICES - -LOG = logging.getLogger("connectivity-tester") - -FALLBACK_TCP_TIMEOUT_S = 3.0 -HTTP_CONNECT_TIMEOUT_S = 5.0 - - -def _socket_path() -> str: - return os.environ.get("DOCKER_SOCKET", "/var/run/docker.sock") - - -def _self_container(client: docker.DockerClient) -> Container: - hostname = socket.gethostname() - for c in client.containers.list(): - cn = c.name.lstrip("/") - hn = c.attrs.get("Config", {}).get("Hostname", "") - if hostname == cn or hostname == hn or hostname[:12] == c.id[:12]: - return c - candidates = [c for c in client.containers.list() if hostname[:12] == c.id] - if len(candidates) == 1: - return candidates[0] - raise RuntimeError(f"self container not found for hostname={hostname}") - - -def _swarm_networks_for(client: docker.DockerClient, - names: Iterable[str]) -> dict: - out = {} - by_name = {n.name: n for n in client.networks.list()} - for name in names: - net = by_name.get(name) - if net is not None and net.attrs.get("Scope") == "swarm": - out[name] = net - return out - - -def _attach(client: docker.DockerClient, container: Container, - network: Network) -> bool: - current = (container.attrs.get("NetworkSettings", {}) or {}).get("Networks", {}) - if network.name in current: - return False - try: - network.connect(container, aliases=["connectivity-tester", - socket.gethostname()]) - return True - except Exception as exc: - if "already exists" in str(exc).lower(): - return False - raise - - -def _detach(client: docker.DockerClient, container: Container, - network: Network) -> None: - try: - network.disconnect(container, force=True) - except Exception as exc: - if "not found" in str(exc).lower(): - return - raise - - -async def _tcp_probe(host: str, port: int, timeout: float) -> Tuple[bool, str]: - try: - fut = asyncio.open_connection(host=host, port=port) - reader, writer = await asyncio.wait_for(fut, timeout=timeout) - writer.close() - try: - await writer.wait_closed() - except Exception: - pass - return True, "tcp ok" - except asyncio.TimeoutError: - return False, "timeout" - except OSError as exc: - return False, f"{type(exc).__name__}: {exc}" - - -async def _proxy_connect(endpoint: Endpoint, timeout: float) -> Tuple[bool, str]: - if endpoint.proxy is None: - return await _tcp_probe(endpoint.name, endpoint.port, timeout=timeout) - proxy_host, proxy_port = endpoint.proxy - target = endpoint.name - target_port = endpoint.port - try: - fut = asyncio.open_connection(host=proxy_host, port=proxy_port) - reader, writer = await asyncio.wait_for(fut, timeout=timeout) - req = ( - f"CONNECT {target}:{target_port} HTTP/1.1\r\n" - f"Host: {target}:{target_port}\r\n\r\n" - ) - writer.write(req.encode("ascii")) - await writer.drain() - status_line = await asyncio.wait_for(reader.readline(), - timeout=timeout) - if not status_line.startswith(b"HTTP/") or b" 200 " not in status_line: - writer.close() - try: - await writer.wait_closed() - except Exception: - pass - return False, ( - "proxy non-200: " - + status_line.strip().decode(errors="replace") - ) - while True: - line = await asyncio.wait_for(reader.readline(), timeout=timeout) - if line in (b"\r\n", b"", b"\n"): - break - writer.close() - try: - await writer.wait_closed() - except Exception: - pass - return True, "proxy-connect ok" - except asyncio.TimeoutError: - return False, "timeout" - except OSError as exc: - return False, f"{type(exc).__name__}: {exc}" - - -async def _check_endpoint(endpoint: Endpoint, timeout: float) -> bool: - if endpoint.proxy is not None: - ok, detail = await _proxy_connect(endpoint, timeout) - else: - ok, detail = await _tcp_probe(endpoint.name, endpoint.port, - timeout=timeout) - label = f"{endpoint.name}:{endpoint.port}" - src = f"via {endpoint.proxy}" if endpoint.proxy else "direct" - if ok: - LOG.info("PASS %s %s %s", label, src, detail) - else: - LOG.error("FAIL %s %s %s", label, src, detail) - return ok - - -async def _check_network(network_name: str, - endpoints: List[Endpoint], - timeout: float) -> Tuple[Set[Endpoint], Set[Endpoint]]: - reachable: Set[Endpoint] = set() - unreachable: Set[Endpoint] = set() - for ep in endpoints: - ok = await _check_endpoint(ep, timeout) - (reachable if ok else unreachable).add(ep) - return reachable, unreachable - - -def _cross_validate(network_name: str, - mapped: List[Endpoint], - reachable: Set[Endpoint], - unreachable: Set[Endpoint], - global_set: Set[Endpoint]) -> None: - mapped_set = set(mapped) - reachable_global = reachable & global_set - for ep in reachable_global - mapped_set: - LOG.error("LEAK-UNEXPECTED %s reachable in %s but not in map", - f"{ep.name}:{ep.port}", network_name) - for ep in mapped_set & unreachable: - LOG.error("MISSING-EXPECTED %s unreachable in %s but listed in map", - f"{ep.name}:{ep.port}", network_name) - for ep in mapped_set & reachable: - LOG.info("OK-MAPPED %s ok in %s", f"{ep.name}:{ep.port}", network_name) - for ep in mapped_set - reachable - unreachable: - LOG.warning("NOT-CHECKED %s in map for %s", - f"{ep.name}:{ep.port}", network_name) - - -def _consistency_audit(net_names: List[str]) -> None: - flat: Set[Endpoint] = set() - for n in net_names: - flat.update(NETWORK_MAP.get(n, [])) - covered = flat | set(PUBLIC_SERVICES) - all_global = set(GLOBAL_SERVICES) - uncovered = all_global - covered - if uncovered: - LOG.warning("GLOBAL-UNCOVERED %s not present in any map", - sorted(f"{e.name}:{e.port}" for e in uncovered)) - - -async def _check_negative(endpoints: List[Endpoint], - timeout: float) -> int: - leaks = 0 - for ep in endpoints: - ok = await _tcp_probe(ep.name, ep.port, timeout=timeout) - if ok: - LOG.error("LEAK %s:%d reachable but must NOT be", - ep.name, ep.port) - leaks += 1 - else: - LOG.info("OK-NEG %s:%d blocked", ep.name, ep.port) - return leaks - - -async def main() -> int: - logging.basicConfig( - level=logging.INFO, - format="%(asctime)s %(levelname)s %(message)s", - stream=sys.stdout, - ) - period = int(os.environ.get("PERIOD_SECONDS", "600")) - timeout = float(os.environ.get("CHECK_TIMEOUT_S", "3")) - LOG.info("starting; period=%ds timeout=%.1fs", period, timeout) - - client = docker.DockerClient(base_url="unix://" + _socket_path()) - self_container = _self_container(client) - - nets = _swarm_networks_for(client, NETWORK_MAP.keys()) - planned = list(NETWORK_MAP.keys()) - LOG.info("configured networks: %d, present in swarm: %d", - len(planned), len(nets)) - for n in nets: - LOG.info("plan network: %s", n) - - _consistency_audit(planned) - - global_set: Set[Endpoint] = set(GLOBAL_SERVICES) - - while True: - started = time.monotonic() - try: - for net_name, net in nets.items(): - eps_in_map: List[Endpoint] = [ - e for e in NETWORK_MAP[net_name] if e.proxy is None - ] - LOG.info("--- round begin, attaching to %s (%d endpoints) ---", - net_name, len(eps_in_map)) - _attach(client, self_container, net) - try: - reachable, unreachable = await _check_network( - net_name, eps_in_map, timeout) - _cross_validate(net_name, eps_in_map, - reachable, unreachable, global_set) - finally: - _detach(client, self_container, net) - LOG.info("--- round end on %s ---", net_name) - - LOG.info("--- public checks (%d endpoints) ---", - len(PUBLIC_SERVICES)) - for ep in PUBLIC_SERVICES: - await _check_endpoint(ep, max(timeout, HTTP_CONNECT_TIMEOUT_S)) - - LOG.info("--- negative checks (%d endpoints) ---", len(NEGATIVE)) - await _check_negative(NEGATIVE, timeout) - except Exception: - LOG.exception("round crashed") - - elapsed = time.monotonic() - started - sleep_for = max(0.0, period - elapsed) - await asyncio.sleep(sleep_for) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/connectivity_tester/network_map.py b/connectivity_tester/network_map.py deleted file mode 100644 index fe57539..0000000 --- a/connectivity_tester/network_map.py +++ /dev/null @@ -1,122 +0,0 @@ -from dataclasses import dataclass -from typing import Dict, List, Optional, Tuple - - -@dataclass(frozen=True) -class Endpoint: - name: str - port: int - proxy: Optional[Tuple[str, int]] = None - tls: bool = False - - -SWARM_NETWORKS = ( - "hermes_infra_hermes_proxy", - "hermes_infra_squid_egress", - "hermes_infra_admin", - "hermes_infra_default", - "hermes_hermes", - "gitea_gitea", - "gmail_mcp_default", - "immich_default", - "torrserv_proxy_default", - "docker_socket_proxy_hermes_proxy", - "connectivity_tester_default", - "brencher_brencher-net", - "brencher2_brencher-net", -) - - -SQUID_PROXY: Tuple[str, int] = ("squid", 3128) - - -NETWORK_MAP: Dict[str, List[Endpoint]] = { - "hermes_infra_hermes_proxy": [ - Endpoint("hermes-gateway", 8000), - Endpoint("hermes-ui", 8087), - Endpoint("squid", 3128), - Endpoint("docker_socket_proxy", 2375), - Endpoint("brencher-state-proxy", 80), - ], - "hermes_infra_squid_egress": [ - Endpoint("squid", 3128), - ], - "hermes_infra_admin": [ - Endpoint("sarg", 80), - ], - "hermes_infra_default": [ - Endpoint("hermes-ui", 8087), - ], - "hermes_hermes": [ - Endpoint("hermes-agent", 8000), - Endpoint("hermes-gateway", 8000), - ], - "gitea_gitea": [ - Endpoint("gitea-server", 3000), - Endpoint("act_runner", 8088), - Endpoint("cache_server", 37051), - ], - "gmail_mcp_default": [ - Endpoint("gmail-mcp-server", 8000), - Endpoint("telegram-mcp-server", 8000), - ], - "immich_default": [ - Endpoint("immich-server", 2283), - Endpoint("immich-database", 5432), - Endpoint("immich-redis", 6379), - Endpoint("immich-machine-learning", 8501), - ], - "torrserv_proxy_default": [ - Endpoint("fileserver", 8000), - Endpoint("reverseproxy", 8080), - ], - "docker_socket_proxy_hermes_proxy": [ - Endpoint("hermes-gateway", 8000), - ], - "connectivity_tester_default": [ - Endpoint("connectivity-tester", 8765), - ], - "brencher_brencher-net": [ - Endpoint("brencher-backend", 5000), - ], - "brencher2_brencher-net": [ - Endpoint("brencher-backend", 5000), - ], -} - - -PUBLIC_SERVICES: List[Endpoint] = [ - Endpoint("gitea.rudolf.keenetic.link", 443, proxy=SQUID_PROXY, tls=True), - Endpoint("gitea.rudolf.keenetic.link", 80, proxy=SQUID_PROXY), - Endpoint("registry.rudolf.keenetic.link", 443, proxy=SQUID_PROXY, tls=True), - Endpoint("registry.rudolf.keenetic.link", 5000, proxy=SQUID_PROXY), - Endpoint("deb.debian.org", 80, proxy=SQUID_PROXY), - Endpoint("deb.debian.org", 443, proxy=SQUID_PROXY, tls=True), - Endpoint("github.com", 443, proxy=SQUID_PROXY, tls=True), - Endpoint("githubusercontent.com", 443, proxy=SQUID_PROXY, tls=True), - Endpoint("telegram.org", 443, proxy=SQUID_PROXY, tls=True), - Endpoint("t.me", 443, proxy=SQUID_PROXY, tls=True), - Endpoint("minimax.io", 443, proxy=SQUID_PROXY, tls=True), - Endpoint("minimax.chat", 443, proxy=SQUID_PROXY, tls=True), - Endpoint("minimax.com", 443, proxy=SQUID_PROXY, tls=True), - Endpoint("pypi.org", 443, proxy=SQUID_PROXY, tls=True), - Endpoint("google.com", 443, proxy=SQUID_PROXY, tls=True), - Endpoint("cloudflare.com", 443, proxy=SQUID_PROXY, tls=True), - Endpoint("wikipedia.org", 443, proxy=SQUID_PROXY, tls=True), - Endpoint("mozilla.org", 443, proxy=SQUID_PROXY, tls=True), - Endpoint("1.1.1.1", 443, proxy=SQUID_PROXY, tls=True), - Endpoint("8.8.8.8", 53), - Endpoint("8.8.4.4", 53), -] - - -GLOBAL_SERVICES: List[Endpoint] = sorted( - {ep for eps in NETWORK_MAP.values() for ep in eps} | set(PUBLIC_SERVICES), - key=lambda e: (e.name, e.port), -) - - -NEGATIVE: List[Endpoint] = [ - Endpoint("169.254.169.254", 80), - Endpoint("metadata.google.internal", 80), -] diff --git a/connectivity_tester/stack-compose.yml b/connectivity_tester/stack-compose.yml deleted file mode 100644 index de80b49..0000000 --- a/connectivity_tester/stack-compose.yml +++ /dev/null @@ -1,25 +0,0 @@ -services: - connectivity-tester: - image: gitea.rudolf.keenetic.link/hermes/connectivity_tester:${version} - hostname: connectivity-tester - user: "0:0" - volumes: - - /var/run/docker.sock:/var/run/docker.sock - healthcheck: - test: ["CMD-SHELL", "test -S /var/run/docker.sock"] - interval: 60s - timeout: 5s - retries: 3 - start_period: 30s - deploy: - mode: global - resources: - limits: - memory: 128M - pids: 64 - reservations: - memory: 64M - restart_policy: - condition: on-failure - delay: 10s - max_attempts: 3