diff --git a/connectivity_tester/Dockerfile b/connectivity_tester/Dockerfile new file mode 100644 index 0000000..5d52445 --- /dev/null +++ b/connectivity_tester/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3.12-slim + +WORKDIR /app +COPY connectivity_tester/ /app/ + +ENV PYTHONUNBUFFERED=1 \ + PERIOD_SECONDS=600 \ + CHECK_TIMEOUT_S=3 + +CMD ["python", "-m", "app"] diff --git a/connectivity_tester/app.py b/connectivity_tester/app.py new file mode 100644 index 0000000..8795a83 --- /dev/null +++ b/connectivity_tester/app.py @@ -0,0 +1,120 @@ +"""Connectivity tester: verifies internal/external endpoints are reachable +and negative-list endpoints are not. Logs structured output, alerts on +unexpected failures. + +ENV: + EXTRA_ENDPOINTS - comma-separated host:port pairs, ad-hoc checks + PERIOD_SECONDS - sleep between full check rounds (default 600) + CHECK_TIMEOUT_S - per-endpoint timeout in seconds (default 3) +""" +from __future__ import annotations + +import asyncio +import logging +import os +import socket +import sys +import time +from typing import Iterable, List, Tuple + +from endpoints import EXPECTED_EXTERNAL, EXPECTED_INTERNAL, NEGATIVE, Endpoint + +LOG = logging.getLogger("connectivity-tester") + + +def _parse_extra(raw: str) -> List[Endpoint]: + out: List[Endpoint] = [] + for token in raw.split(","): + token = token.strip() + if not token: + continue + if ":" not in token: + continue + host, port_s = token.rsplit(":", 1) + try: + port = int(port_s) + except ValueError: + continue + out.append((host, port, "extra")) + return out + + +async def _tcp_check(host: str, port: int, timeout: float) -> Tuple[bool, str]: + try: + fut = asyncio.open_connection(host, port) + reader, writer = await asyncio.wait_for(fut, timeout=timeout) + writer.close() + try: + await writer.wait_closed() + except Exception: + pass + return True, "tcp ok" + except asyncio.TimeoutError: + return False, "timeout" + except OSError as exc: + return False, f"{type(exc).__name__}: {exc}" + + +async def _check_positive(endpoints: Iterable[Endpoint], timeout: float) -> int: + failures = 0 + for host, port, label in endpoints: + ok, detail = await _tcp_check(host, port, timeout) + if ok: + LOG.info("PASS %s:%d (%s) %s", host, port, label, detail) + else: + LOG.error("FAIL %s:%d (%s) %s", host, port, label, detail) + failures += 1 + return failures + + +async def _check_negative(endpoints: Iterable[Endpoint], timeout: float) -> int: + leaks = 0 + for host, port, label in endpoints: + ok, detail = await _tcp_check(host, port, timeout) + if ok: + LOG.error( + "LEAK %s:%d (%s) reachable but must NOT be: %s", + host, port, label, detail, + ) + leaks += 1 + else: + LOG.info("OK-NEG %s:%d (%s) blocked: %s", host, port, label, detail) + return leaks + + +async def _round(timeout: float) -> int: + extras = _parse_extra(os.environ.get("EXTRA_ENDPOINTS", "")) + LOG.info("--- round begin, %d internal + %d external + %d extra + %d negative ---", + len(EXPECTED_INTERNAL), len(EXPECTED_EXTERNAL), len(extras), len(NEGATIVE)) + failures = 0 + failures += await _check_positive(EXPECTED_INTERNAL, timeout) + failures += await _check_positive(EXPECTED_EXTERNAL, timeout) + failures += await _check_positive(extras, timeout) + leaks = await _check_negative(NEGATIVE, timeout) + failures += leaks + LOG.info("--- round end, %d failure(s) ---", failures) + return failures + + +async def main() -> int: + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(message)s", + stream=sys.stdout, + ) + period = int(os.environ.get("PERIOD_SECONDS", "600")) + timeout = float(os.environ.get("CHECK_TIMEOUT_S", "3")) + LOG.info("starting; period=%ds, check_timeout=%.1fs", period, timeout) + while True: + started = time.monotonic() + try: + await _round(timeout) + except Exception: + LOG.exception("round crashed") + elapsed = time.monotonic() - started + sleep_for = max(0.0, period - elapsed) + await asyncio.sleep(sleep_for) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/connectivity_tester/endpoints.py b/connectivity_tester/endpoints.py new file mode 100644 index 0000000..66e7300 --- /dev/null +++ b/connectivity_tester/endpoints.py @@ -0,0 +1,43 @@ +"""Endpoint allow-list and negative-list for connectivity-tester. + +EXPECTED_INTERNAL: TCP services in overlay networks that MUST be reachable. +EXPECTED_EXTERNAL: external hosts that MUST be reachable via squid egress. +NEGATIVE: hosts that MUST NOT respond (proxy / firewall sanity check). + +Each entry is (host, port, label). +""" +from __future__ import annotations + +from typing import List, Tuple + +Endpoint = Tuple[str, int, str] + + +EXPECTED_INTERNAL: List[Endpoint] = [ + ("squid", 3128, "egress proxy"), + ("hermes-gateway", 8000, "hermes messaging gateway"), + ("hermes-ui", 8087, "hermes dashboard ui"), + ("docker_socket_proxy", 2375, "read-only docker socket proxy"), + ("brencher-state-proxy", 80, "brencher state reverse proxy"), + ("gitea-server", 3000, "gitea http"), + ("act_runner", 8088, "gitea actions runner"), + ("telegram-mcp-server", 8000, "telegram mcp"), + ("gmail-mcp-server", 8000, "gmail mcp"), + ("voice-gateway", 8765, "voice realtime ws bridge"), + ("hermes-proxy", 8645, "openai-compatible http egress"), +] + + +EXPECTED_EXTERNAL: List[Endpoint] = [ + ("huggingface.co", 443, "piper voice model download"), + ("registry.rudolf.keenetic.link", 5000, "local docker registry"), + ("gitea.rudolf.keenetic.link", 443, "public gitea"), + ("pypi.org", 443, "python package index"), + ("github.com", 443, "github api"), +] + + +NEGATIVE: List[Endpoint] = [ + ("169.254.169.254", 80, "cloud metadata service"), + ("metadata.google.internal", 80, "gcp metadata"), +] diff --git a/connectivity_tester/stack-compose.yml b/connectivity_tester/stack-compose.yml new file mode 100644 index 0000000..70dce8d --- /dev/null +++ b/connectivity_tester/stack-compose.yml @@ -0,0 +1,44 @@ +name: connectivity_tester + +services: + connectivity-tester: + build: + context: . + dockerfile: connectivity_tester/Dockerfile + image: registry.rudolf.keenetic.link/connectivity_tester:${version} + hostname: connectivity-tester + user: "0:0" + networks: + - hermes_proxy + - default + environment: + PERIOD_SECONDS: "600" + CHECK_TIMEOUT_S: "3" + EXTRA_ENDPOINTS: "${EXTRA_ENDPOINTS:-}" + HTTP_PROXY: "http://squid:3128" + HTTPS_PROXY: "http://squid:3128" + NO_PROXY: "localhost,127.0.0.1,.local,squid,hermes-gateway,hermes-agent,hermes-proxy,brencher-state-proxy,docker_socket_proxy,10.0.0.0/8" + healthcheck: + test: ["CMD-SHELL", "python -c \"import socket; s=socket.socket(); s.settimeout(2); s.connect(('squid', 3128)); s.close()\""] + interval: 60s + timeout: 5s + retries: 3 + start_period: 30s + deploy: + mode: global + resources: + limits: + memory: 128M + pids: 64 + reservations: + memory: 64M + restart_policy: + condition: on-failure + delay: 10s + max_attempts: 3 + +networks: + hermes_proxy: + external: true + default: + external: true