diff --git a/connectivity_tester/Dockerfile b/connectivity_tester/Dockerfile index 85943a9..65dc943 100644 --- a/connectivity_tester/Dockerfile +++ b/connectivity_tester/Dockerfile @@ -3,6 +3,8 @@ FROM python:3.12-slim WORKDIR /app COPY . /app/ +RUN pip install --no-cache-dir docker==7.1.0 + ENV PYTHONUNBUFFERED=1 \ PERIOD_SECONDS=600 \ CHECK_TIMEOUT_S=3 diff --git a/connectivity_tester/app.py b/connectivity_tester/app.py index 8795a83..79e213e 100644 --- a/connectivity_tester/app.py +++ b/connectivity_tester/app.py @@ -1,11 +1,10 @@ -"""Connectivity tester: verifies internal/external endpoints are reachable -and negative-list endpoints are not. Logs structured output, alerts on -unexpected failures. +"""Connectivity tester. -ENV: - EXTRA_ENDPOINTS - comma-separated host:port pairs, ad-hoc checks - PERIOD_SECONDS - sleep between full check rounds (default 600) - CHECK_TIMEOUT_S - per-endpoint timeout in seconds (default 3) +For every network in network_map.NETWORK_MAP the container attaches itself, +runs TCP checks against the listed services, detaches, and proceeds to the +next network. Cross-validates GLOBAL_SERVICES against each network map: logs +errors when reachable services are absent from the map or when expected +services are unreachable. """ from __future__ import annotations @@ -15,30 +14,86 @@ import os import socket import sys import time -from typing import Iterable, List, Tuple +from typing import Iterable, List, Set, Tuple -from endpoints import EXPECTED_EXTERNAL, EXPECTED_INTERNAL, NEGATIVE, Endpoint +import docker +from docker.models.containers import Container +from docker.models.networks import Network + +from network_map import GLOBAL_SERVICES, NETWORK_MAP, Service LOG = logging.getLogger("connectivity-tester") -def _parse_extra(raw: str) -> List[Endpoint]: - out: List[Endpoint] = [] - for token in raw.split(","): - token = token.strip() - if not token: +def _socket_path() -> str: + return os.environ.get("DOCKER_SOCKET", "/var/run/docker.sock") + + +def _self_container(client: docker.DockerClient) -> Container: + hostname = socket.gethostname() + for c in client.containers.list(): + if (c.attrs.get("Config", {}).get("Hostname") == hostname + or c.name.lstrip("/") == hostname): + return c + candidates = [c for c in client.containers.list() + if hostname[:12] in c.id] + if len(candidates) == 1: + return candidates[0] + raise RuntimeError(f"self container not found for hostname={hostname}") + + +def _filtered_networks(client: docker.DockerClient) -> List[Network]: + keep_prefixes = ( + "hermes_infra_", "gitea_", "gmail_mcp_", "telegram_", + "hermes_", "brencher", "registry_", "immich_", + "torrserv_proxy_", "photosHelper_", "nginx_", + "docker_socket_proxy_", + ) + out = [] + for n in client.networks.list(): + if n.attrs.get("Scope") != "swarm": continue - if ":" not in token: + name = n.name + if name in ("ingress",): continue - host, port_s = token.rsplit(":", 1) - try: - port = int(port_s) - except ValueError: - continue - out.append((host, port, "extra")) + if any(name.startswith(p) for p in keep_prefixes): + out.append(n) return out +def _build_map_from_running(networks: Iterable[Network]) -> dict: + """Reverse-engineer NAME -> swarm network name from running container endpoints.""" + out = {} + for net in networks: + endpoint_count = len(net.attrs.get("Containers") or {}) + out[net.name] = {"endpoints": endpoint_count, "id": net.id} + return out + + +def _attach(client: docker.DockerClient, container: Container, + network: Network) -> None: + if network.name in (container.attrs.get("NetworkSettings", {}) + .get("Networks", {}).keys()): + return + try: + network.connect(container, aliases=["connectivity-tester", + socket.gethostname()]) + except Exception as exc: + if "already exists" in str(exc).lower(): + return + raise + + +def _detach(client: docker.DockerClient, container: Container, + network: Network) -> None: + try: + network.disconnect(container, force=True) + except Exception as exc: + if "not found" in str(exc).lower(): + return + raise + + async def _tcp_check(host: str, port: int, timeout: float) -> Tuple[bool, str]: try: fut = asyncio.open_connection(host, port) @@ -55,45 +110,60 @@ async def _tcp_check(host: str, port: int, timeout: float) -> Tuple[bool, str]: return False, f"{type(exc).__name__}: {exc}" -async def _check_positive(endpoints: Iterable[Endpoint], timeout: float) -> int: - failures = 0 - for host, port, label in endpoints: - ok, detail = await _tcp_check(host, port, timeout) - if ok: - LOG.info("PASS %s:%d (%s) %s", host, port, label, detail) - else: - LOG.error("FAIL %s:%d (%s) %s", host, port, label, detail) - failures += 1 - return failures +async def _check_one(host: str, port: int, label: str, + timeout: float) -> bool: + ok, detail = await _tcp_check(host, port, timeout) + if ok: + LOG.info("PASS %s:%d (%s) %s", host, port, label, detail) + else: + LOG.error("FAIL %s:%d (%s) %s", host, port, label, detail) + return ok -async def _check_negative(endpoints: Iterable[Endpoint], timeout: float) -> int: - leaks = 0 - for host, port, label in endpoints: - ok, detail = await _tcp_check(host, port, timeout) - if ok: - LOG.error( - "LEAK %s:%d (%s) reachable but must NOT be: %s", - host, port, label, detail, - ) - leaks += 1 - else: - LOG.info("OK-NEG %s:%d (%s) blocked: %s", host, port, label, detail) - return leaks +async def _check_network(network: Network, services: List[Service], + timeout: float) -> Tuple[Set[Service], Set[Service]]: + reachable: Set[Service] = set() + unreachable: Set[Service] = set() + for host, port in services: + ok = await _check_one(host, port, network.name, timeout) + (reachable if ok else unreachable).add((host, port)) + return reachable, unreachable -async def _round(timeout: float) -> int: - extras = _parse_extra(os.environ.get("EXTRA_ENDPOINTS", "")) - LOG.info("--- round begin, %d internal + %d external + %d extra + %d negative ---", - len(EXPECTED_INTERNAL), len(EXPECTED_EXTERNAL), len(extras), len(NEGATIVE)) - failures = 0 - failures += await _check_positive(EXPECTED_INTERNAL, timeout) - failures += await _check_positive(EXPECTED_EXTERNAL, timeout) - failures += await _check_positive(extras, timeout) - leaks = await _check_negative(NEGATIVE, timeout) - failures += leaks - LOG.info("--- round end, %d failure(s) ---", failures) - return failures +def _discover_attached(client: docker.DockerClient, + container: Container) -> Set[str]: + return set((container.attrs.get("NetworkSettings", {}) or {}) + .get("Networks", {}).keys()) + + +def _cross_validate(network_name: str, + mapped: List[Service], + reachable: Set[Service], + unreachable: Set[Service], + all_global: List[Service]) -> None: + mapped_set = set(mapped) + reachable_global = reachable & set(all_global) + for svc in reachable_global - mapped_set: + LOG.error("LEAK-UNEXPECTED %s reachable in %s but not in map", + f"{svc[0]}:{svc[1]}", network_name) + for svc in mapped_set & unreachable: + LOG.error("MISSING-EXPECTED %s unreachable in %s but listed in map", + f"{svc[0]}:{svc[1]}", network_name) + for svc in mapped_set & reachable: + LOG.info("OK-MAPPED %s ok in %s", f"{svc[0]}:{svc[1]}", network_name) + for svc in mapped_set - reachable - unreachable: + LOG.warning("NOT-CHECKED %s in map for %s", + f"{svc[0]}:{svc[1]}", network_name) + + +def _consistency_audit(networks_names: List[str], all_global: List[Service]) -> None: + flat: Set[Service] = set() + for n in networks_names: + flat.update(NETWORK_MAP.get(n, [])) + leftover = set(all_global) - flat + if leftover: + LOG.warning("GLOBAL-UNCOVERED %s not present in any map", + sorted(f"{s[0]}:{s[1]}" for s in leftover)) async def main() -> int: @@ -104,13 +174,44 @@ async def main() -> int: ) period = int(os.environ.get("PERIOD_SECONDS", "600")) timeout = float(os.environ.get("CHECK_TIMEOUT_S", "3")) - LOG.info("starting; period=%ds, check_timeout=%.1fs", period, timeout) + LOG.info("starting; period=%ds timeout=%.1fs", period, timeout) + + client = docker.DockerClient(base_url="unix://" + _socket_path()) + self_container = _self_container(client) + self_id_short = self_container.id[:12] + + networks_docker = _filtered_networks(client) + networks_known = [n for n in networks_docker if n.name in NETWORK_MAP] + LOG.info("configured networks: %d, present in swarm: %d", + len(NETWORK_MAP), len(networks_known)) + for n in networks_known: + LOG.info("plan network: %s", n.name) + + _consistency_audit([n.name for n in networks_known], GLOBAL_SERVICES) + while True: started = time.monotonic() + failures = 0 try: - await _round(timeout) + for network in networks_known: + LOG.info("--- round begin, attaching to %s ---", network.name) + _attach(client, self_container, network) + try: + reachable, unreachable = await _check_network( + network, NETWORK_MAP[network.name], timeout) + _cross_validate( + network.name, + NETWORK_MAP[network.name], + reachable, + unreachable, + GLOBAL_SERVICES, + ) + finally: + _detach(client, self_container, network) + LOG.info("--- round end on %s ---", network.name) except Exception: LOG.exception("round crashed") + elapsed = time.monotonic() - started sleep_for = max(0.0, period - elapsed) await asyncio.sleep(sleep_for) diff --git a/connectivity_tester/endpoints.py b/connectivity_tester/endpoints.py deleted file mode 100644 index e6f63ae..0000000 --- a/connectivity_tester/endpoints.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Endpoint allow-list and negative-list for connectivity-tester. - -EXPECTED_INTERNAL: TCP services in overlay networks that MUST be reachable. -EXPECTED_EXTERNAL: external hosts that MUST be reachable via squid egress. -NEGATIVE: hosts that MUST NOT respond (proxy / firewall sanity check). - -Each entry is (host, port, label). -""" -from __future__ import annotations - -from typing import List, Tuple - -Endpoint = Tuple[str, int, str] - - -EXPECTED_INTERNAL: List[Endpoint] = [ - ("squid", 3128, "egress proxy"), - ("hermes-gateway", 8000, "hermes messaging gateway"), - ("hermes-ui", 8087, "hermes dashboard ui"), - ("docker_socket_proxy", 2375, "read-only docker socket proxy"), - ("brencher-state-proxy", 80, "brencher state reverse proxy"), - ("gitea-server", 3000, "gitea http"), - ("act_runner", 8088, "gitea actions runner"), - ("telegram-mcp-server", 8000, "telegram mcp"), - ("gmail-mcp-server", 8000, "gmail mcp"), - ("voice-gateway", 8765, "voice realtime ws bridge"), - ("hermes-proxy", 8645, "openai-compatible http egress"), -] - - -EXPECTED_EXTERNAL: List[Endpoint] = [ - ("huggingface.co", 443, "piper voice model download"), - ("registry.rudolf.keenetic.link", 5000, "local docker registry"), - ("gitea.rudolf.keenetic.link", 443, "public gitea"), - ("pypi.org", 443, "python package index"), - ("github.com", 443, "github api"), - ("google.com", 443, "egress sanity check"), - ("192.168.1.1", 80, "lan gateway .1"), - ("192.168.0.1", 80, "lan gateway .0"), -] - - -NEGATIVE: List[Endpoint] = [ - ("169.254.169.254", 80, "cloud metadata service"), - ("metadata.google.internal", 80, "gcp metadata"), -] diff --git a/connectivity_tester/network_map.py b/connectivity_tester/network_map.py new file mode 100644 index 0000000..eb1b6d5 --- /dev/null +++ b/connectivity_tester/network_map.py @@ -0,0 +1,40 @@ +from typing import Dict, List, Tuple + +Service = Tuple[str, int] + +GLOBAL_SERVICES: List[Service] = [ + ("hermes-gateway", 8000), + ("hermes-agent", 8000), + ("hermes-ui", 8087), + ("squid", 3128), + ("gitea-server", 3000), + ("act_runner", 8088), + ("brencher-state-proxy", 80), + ("docker_socket_proxy", 2375), + ("telegram-mcp-server", 8000), + ("gmail-mcp-server", 8000), + ("voice-gateway", 8765), + ("hermes-proxy", 8645), +] + +NETWORK_MAP: Dict[str, List[Service]] = { + "hermes_infra_hermes_proxy": [ + ("hermes-gateway", 8000), + ("hermes-agent", 8000), + ("hermes-ui", 8087), + ("squid", 3128), + ("gitea-server", 3000), + ("act_runner", 8088), + ("brencher-state-proxy", 80), + ("docker_socket_proxy", 2375), + ("telegram-mcp-server", 8000), + ("gmail-mcp-server", 8000), + ("voice-gateway", 8765), + ("hermes-proxy", 8645), + ], + "gitea_gitea": [ + ("gitea-server", 3000), + ("act_runner", 8088), + ("cache_server", 37051), + ], +} diff --git a/connectivity_tester/stack-compose.yml b/connectivity_tester/stack-compose.yml index e654336..b69c92e 100644 --- a/connectivity_tester/stack-compose.yml +++ b/connectivity_tester/stack-compose.yml @@ -6,18 +6,10 @@ services: image: registry.rudolf.keenetic.link/connectivity_tester:${version} hostname: connectivity-tester user: "0:0" - networks: - - hermes_proxy - - default - environment: - PERIOD_SECONDS: "600" - CHECK_TIMEOUT_S: "3" - EXTRA_ENDPOINTS: "" - HTTP_PROXY: "http://squid:3128" - HTTPS_PROXY: "http://squid:3128" - NO_PROXY: "localhost,127.0.0.1,.local,squid,hermes-gateway,hermes-agent,hermes-proxy,brencher-state-proxy,docker_socket_proxy,10.0.0.0/8" + volumes: + - /var/run/docker.sock:/var/run/docker.sock healthcheck: - test: ["CMD-SHELL", "python -c \"import socket; s=socket.socket(); s.settimeout(2); s.connect(('squid', 3128)); s.close()\""] + test: ["CMD-SHELL", "test -S /var/run/docker.sock"] interval: 60s timeout: 5s retries: 3 @@ -34,9 +26,3 @@ services: condition: on-failure delay: 10s max_attempts: 3 - -networks: - hermes_proxy: - external: true - default: - external: true