From 3dbd0db8b6905a9c0aa2405adc50a4df46c93ba1 Mon Sep 17 00:00:00 2001 From: rudolf Date: Sat, 10 Oct 2026 11:43:00 +0200 Subject: [PATCH] x --- .gitea/workflows/release.yml | 49 +++++++ Dockerfile | 12 ++ app.py | 276 +++++++++++++++++++++++++++++++++++ empty.txy | 1 - network_map.py | 122 ++++++++++++++++ stack-compose.yml | 28 ++++ 6 files changed, 487 insertions(+), 1 deletion(-) create mode 100644 .gitea/workflows/release.yml create mode 100644 Dockerfile create mode 100644 app.py delete mode 100644 empty.txy create mode 100644 network_map.py create mode 100644 stack-compose.yml diff --git a/.gitea/workflows/release.yml b/.gitea/workflows/release.yml new file mode 100644 index 0000000..acc3a33 --- /dev/null +++ b/.gitea/workflows/release.yml @@ -0,0 +1,49 @@ +name: release + +on: + push: + branches: + - 'auto-*' + tags: + - 'v*' + workflow_dispatch: + inputs: + tag: + description: 'Image tag (e.g. auto-)' + required: false + default: latest + type: string + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: log into Gitea Packages + uses: docker/login-action@v3 + with: + registry: gitea.rudolf.keenetic.link + username: ${{ secrets.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_TOKEN }} + + - name: resolve version + id: version + run: | + if [ -n "${{ inputs.tag }}" ] && [ "${{ inputs.tag }}" != "latest" ]; then + echo "VERSION=${{ inputs.tag }}" >> "$GITHUB_OUTPUT" + else + echo "VERSION=${GITHUB_REF_NAME##*/}" >> "$GITHUB_OUTPUT" + fi + echo "VERSION=$VERSION" + + - name: build and push + uses: docker/build-push-action@v6 + with: + context: . + file: Dockerfile + push: true + tags: | + gitea.rudolf.keenetic.link/Org1/swarm_connectivity_tester:${{ steps.version.outputs.VERSION }} + gitea.rudolf.keenetic.link/Org1/swarm_connectivity_tester:latest diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..65dc943 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,12 @@ +FROM python:3.12-slim + +WORKDIR /app +COPY . /app/ + +RUN pip install --no-cache-dir docker==7.1.0 + +ENV PYTHONUNBUFFERED=1 \ + PERIOD_SECONDS=600 \ + CHECK_TIMEOUT_S=3 + +CMD ["python", "-m", "app"] diff --git a/app.py b/app.py new file mode 100644 index 0000000..e34c6e3 --- /dev/null +++ b/app.py @@ -0,0 +1,276 @@ +"""Connectivity tester. + +For every configured overlay network the container attaches itself, runs +TCP checks against its endpoints, then detaches. Endpoints marked with +``proxy=(host, port)`` are probed through that HTTP proxy (CONNECT). After +each round a final loop checks PUBLIC_SERVICES. Cross-validation: +- service reachable but not in map -> ERROR 'LEAK-UNEXPECTED' +- service in map but unreachable -> ERROR 'MISSING-EXPECTED' +- global service not in any map -> WARNING 'GLOBAL-UNCOVERED' +""" +from __future__ import annotations + +import asyncio +import logging +import os +import socket +import sys +import time +from typing import Iterable, List, Set, Tuple + +import docker +from docker.models.containers import Container +from docker.models.networks import Network + +from network_map import Endpoint, GLOBAL_SERVICES, NEGATIVE, NETWORK_MAP, PUBLIC_SERVICES + +LOG = logging.getLogger("connectivity-tester") + +FALLBACK_TCP_TIMEOUT_S = 3.0 +HTTP_CONNECT_TIMEOUT_S = 5.0 + + +def _socket_path() -> str: + return os.environ.get("DOCKER_SOCKET", "/var/run/docker.sock") + + +def _self_container(client: docker.DockerClient) -> Container: + hostname = socket.gethostname() + for c in client.containers.list(): + cn = c.name.lstrip("/") + hn = c.attrs.get("Config", {}).get("Hostname", "") + if hostname == cn or hostname == hn or hostname[:12] == c.id[:12]: + return c + candidates = [c for c in client.containers.list() if hostname[:12] == c.id] + if len(candidates) == 1: + return candidates[0] + raise RuntimeError(f"self container not found for hostname={hostname}") + + +def _swarm_networks_for(client: docker.DockerClient, + names: Iterable[str]) -> dict: + out = {} + by_name = {n.name: n for n in client.networks.list()} + for name in names: + net = by_name.get(name) + if net is not None and net.attrs.get("Scope") == "swarm": + out[name] = net + return out + + +def _attach(client: docker.DockerClient, container: Container, + network: Network) -> bool: + current = (container.attrs.get("NetworkSettings", {}) or {}).get("Networks", {}) + if network.name in current: + return False + try: + network.connect(container, aliases=["connectivity-tester", + socket.gethostname()]) + return True + except Exception as exc: + if "already exists" in str(exc).lower(): + return False + raise + + +def _detach(client: docker.DockerClient, container: Container, + network: Network) -> None: + try: + network.disconnect(container, force=True) + except Exception as exc: + if "not found" in str(exc).lower(): + return + raise + + +async def _tcp_probe(host: str, port: int, timeout: float) -> Tuple[bool, str]: + try: + fut = asyncio.open_connection(host=host, port=port) + reader, writer = await asyncio.wait_for(fut, timeout=timeout) + writer.close() + try: + await writer.wait_closed() + except Exception: + pass + return True, "tcp ok" + except asyncio.TimeoutError: + return False, "timeout" + except OSError as exc: + return False, f"{type(exc).__name__}: {exc}" + + +async def _proxy_connect(endpoint: Endpoint, timeout: float) -> Tuple[bool, str]: + if endpoint.proxy is None: + return await _tcp_probe(endpoint.name, endpoint.port, timeout=timeout) + proxy_host, proxy_port = endpoint.proxy + target = endpoint.name + target_port = endpoint.port + try: + fut = asyncio.open_connection(host=proxy_host, port=proxy_port) + reader, writer = await asyncio.wait_for(fut, timeout=timeout) + req = ( + f"CONNECT {target}:{target_port} HTTP/1.1\r\n" + f"Host: {target}:{target_port}\r\n\r\n" + ) + writer.write(req.encode("ascii")) + await writer.drain() + status_line = await asyncio.wait_for(reader.readline(), + timeout=timeout) + if not status_line.startswith(b"HTTP/") or b" 200 " not in status_line: + writer.close() + try: + await writer.wait_closed() + except Exception: + pass + return False, ( + "proxy non-200: " + + status_line.strip().decode(errors="replace") + ) + while True: + line = await asyncio.wait_for(reader.readline(), timeout=timeout) + if line in (b"\r\n", b"", b"\n"): + break + writer.close() + try: + await writer.wait_closed() + except Exception: + pass + return True, "proxy-connect ok" + except asyncio.TimeoutError: + return False, "timeout" + except OSError as exc: + return False, f"{type(exc).__name__}: {exc}" + + +async def _check_endpoint(endpoint: Endpoint, timeout: float) -> bool: + if endpoint.proxy is not None: + ok, detail = await _proxy_connect(endpoint, timeout) + else: + ok, detail = await _tcp_probe(endpoint.name, endpoint.port, + timeout=timeout) + label = f"{endpoint.name}:{endpoint.port}" + src = f"via {endpoint.proxy}" if endpoint.proxy else "direct" + if ok: + LOG.info("PASS %s %s %s", label, src, detail) + else: + LOG.error("FAIL %s %s %s", label, src, detail) + return ok + + +async def _check_network(network_name: str, + endpoints: List[Endpoint], + timeout: float) -> Tuple[Set[Endpoint], Set[Endpoint]]: + reachable: Set[Endpoint] = set() + unreachable: Set[Endpoint] = set() + for ep in endpoints: + ok = await _check_endpoint(ep, timeout) + (reachable if ok else unreachable).add(ep) + return reachable, unreachable + + +def _cross_validate(network_name: str, + mapped: List[Endpoint], + reachable: Set[Endpoint], + unreachable: Set[Endpoint], + global_set: Set[Endpoint]) -> None: + mapped_set = set(mapped) + reachable_global = reachable & global_set + for ep in reachable_global - mapped_set: + LOG.error("LEAK-UNEXPECTED %s reachable in %s but not in map", + f"{ep.name}:{ep.port}", network_name) + for ep in mapped_set & unreachable: + LOG.error("MISSING-EXPECTED %s unreachable in %s but listed in map", + f"{ep.name}:{ep.port}", network_name) + for ep in mapped_set & reachable: + LOG.info("OK-MAPPED %s ok in %s", f"{ep.name}:{ep.port}", network_name) + for ep in mapped_set - reachable - unreachable: + LOG.warning("NOT-CHECKED %s in map for %s", + f"{ep.name}:{ep.port}", network_name) + + +def _consistency_audit(net_names: List[str]) -> None: + flat: Set[Endpoint] = set() + for n in net_names: + flat.update(NETWORK_MAP.get(n, [])) + covered = flat | set(PUBLIC_SERVICES) + all_global = set(GLOBAL_SERVICES) + uncovered = all_global - covered + if uncovered: + LOG.warning("GLOBAL-UNCOVERED %s not present in any map", + sorted(f"{e.name}:{e.port}" for e in uncovered)) + + +async def _check_negative(endpoints: List[Endpoint], + timeout: float) -> int: + leaks = 0 + for ep in endpoints: + ok = await _tcp_probe(ep.name, ep.port, timeout=timeout) + if ok: + LOG.error("LEAK %s:%d reachable but must NOT be", + ep.name, ep.port) + leaks += 1 + else: + LOG.info("OK-NEG %s:%d blocked", ep.name, ep.port) + return leaks + + +async def main() -> int: + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(message)s", + stream=sys.stdout, + ) + period = int(os.environ.get("PERIOD_SECONDS", "600")) + timeout = float(os.environ.get("CHECK_TIMEOUT_S", "3")) + LOG.info("starting; period=%ds timeout=%.1fs", period, timeout) + + client = docker.DockerClient(base_url="unix://" + _socket_path()) + self_container = _self_container(client) + + nets = _swarm_networks_for(client, NETWORK_MAP.keys()) + planned = list(NETWORK_MAP.keys()) + LOG.info("configured networks: %d, present in swarm: %d", + len(planned), len(nets)) + for n in nets: + LOG.info("plan network: %s", n) + + _consistency_audit(planned) + + global_set: Set[Endpoint] = set(GLOBAL_SERVICES) + + while True: + started = time.monotonic() + try: + for net_name, net in nets.items(): + eps_in_map: List[Endpoint] = [ + e for e in NETWORK_MAP[net_name] if e.proxy is None + ] + LOG.info("--- round begin, attaching to %s (%d endpoints) ---", + net_name, len(eps_in_map)) + _attach(client, self_container, net) + try: + reachable, unreachable = await _check_network( + net_name, eps_in_map, timeout) + _cross_validate(net_name, eps_in_map, + reachable, unreachable, global_set) + finally: + _detach(client, self_container, net) + LOG.info("--- round end on %s ---", net_name) + + LOG.info("--- public checks (%d endpoints) ---", + len(PUBLIC_SERVICES)) + for ep in PUBLIC_SERVICES: + await _check_endpoint(ep, max(timeout, HTTP_CONNECT_TIMEOUT_S)) + + LOG.info("--- negative checks (%d endpoints) ---", len(NEGATIVE)) + await _check_negative(NEGATIVE, timeout) + except Exception: + LOG.exception("round crashed") + + elapsed = time.monotonic() - started + sleep_for = max(0.0, period - elapsed) + await asyncio.sleep(sleep_for) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/empty.txy b/empty.txy deleted file mode 100644 index c1b0730..0000000 --- a/empty.txy +++ /dev/null @@ -1 +0,0 @@ -x \ No newline at end of file diff --git a/network_map.py b/network_map.py new file mode 100644 index 0000000..fe57539 --- /dev/null +++ b/network_map.py @@ -0,0 +1,122 @@ +from dataclasses import dataclass +from typing import Dict, List, Optional, Tuple + + +@dataclass(frozen=True) +class Endpoint: + name: str + port: int + proxy: Optional[Tuple[str, int]] = None + tls: bool = False + + +SWARM_NETWORKS = ( + "hermes_infra_hermes_proxy", + "hermes_infra_squid_egress", + "hermes_infra_admin", + "hermes_infra_default", + "hermes_hermes", + "gitea_gitea", + "gmail_mcp_default", + "immich_default", + "torrserv_proxy_default", + "docker_socket_proxy_hermes_proxy", + "connectivity_tester_default", + "brencher_brencher-net", + "brencher2_brencher-net", +) + + +SQUID_PROXY: Tuple[str, int] = ("squid", 3128) + + +NETWORK_MAP: Dict[str, List[Endpoint]] = { + "hermes_infra_hermes_proxy": [ + Endpoint("hermes-gateway", 8000), + Endpoint("hermes-ui", 8087), + Endpoint("squid", 3128), + Endpoint("docker_socket_proxy", 2375), + Endpoint("brencher-state-proxy", 80), + ], + "hermes_infra_squid_egress": [ + Endpoint("squid", 3128), + ], + "hermes_infra_admin": [ + Endpoint("sarg", 80), + ], + "hermes_infra_default": [ + Endpoint("hermes-ui", 8087), + ], + "hermes_hermes": [ + Endpoint("hermes-agent", 8000), + Endpoint("hermes-gateway", 8000), + ], + "gitea_gitea": [ + Endpoint("gitea-server", 3000), + Endpoint("act_runner", 8088), + Endpoint("cache_server", 37051), + ], + "gmail_mcp_default": [ + Endpoint("gmail-mcp-server", 8000), + Endpoint("telegram-mcp-server", 8000), + ], + "immich_default": [ + Endpoint("immich-server", 2283), + Endpoint("immich-database", 5432), + Endpoint("immich-redis", 6379), + Endpoint("immich-machine-learning", 8501), + ], + "torrserv_proxy_default": [ + Endpoint("fileserver", 8000), + Endpoint("reverseproxy", 8080), + ], + "docker_socket_proxy_hermes_proxy": [ + Endpoint("hermes-gateway", 8000), + ], + "connectivity_tester_default": [ + Endpoint("connectivity-tester", 8765), + ], + "brencher_brencher-net": [ + Endpoint("brencher-backend", 5000), + ], + "brencher2_brencher-net": [ + Endpoint("brencher-backend", 5000), + ], +} + + +PUBLIC_SERVICES: List[Endpoint] = [ + Endpoint("gitea.rudolf.keenetic.link", 443, proxy=SQUID_PROXY, tls=True), + Endpoint("gitea.rudolf.keenetic.link", 80, proxy=SQUID_PROXY), + Endpoint("registry.rudolf.keenetic.link", 443, proxy=SQUID_PROXY, tls=True), + Endpoint("registry.rudolf.keenetic.link", 5000, proxy=SQUID_PROXY), + Endpoint("deb.debian.org", 80, proxy=SQUID_PROXY), + Endpoint("deb.debian.org", 443, proxy=SQUID_PROXY, tls=True), + Endpoint("github.com", 443, proxy=SQUID_PROXY, tls=True), + Endpoint("githubusercontent.com", 443, proxy=SQUID_PROXY, tls=True), + Endpoint("telegram.org", 443, proxy=SQUID_PROXY, tls=True), + Endpoint("t.me", 443, proxy=SQUID_PROXY, tls=True), + Endpoint("minimax.io", 443, proxy=SQUID_PROXY, tls=True), + Endpoint("minimax.chat", 443, proxy=SQUID_PROXY, tls=True), + Endpoint("minimax.com", 443, proxy=SQUID_PROXY, tls=True), + Endpoint("pypi.org", 443, proxy=SQUID_PROXY, tls=True), + Endpoint("google.com", 443, proxy=SQUID_PROXY, tls=True), + Endpoint("cloudflare.com", 443, proxy=SQUID_PROXY, tls=True), + Endpoint("wikipedia.org", 443, proxy=SQUID_PROXY, tls=True), + Endpoint("mozilla.org", 443, proxy=SQUID_PROXY, tls=True), + Endpoint("1.1.1.1", 443, proxy=SQUID_PROXY, tls=True), + Endpoint("8.8.8.8", 53), + Endpoint("8.8.4.4", 53), +] + + +GLOBAL_SERVICES: List[Endpoint] = sorted( + {ep for eps in NETWORK_MAP.values() for ep in eps} | set(PUBLIC_SERVICES), + key=lambda e: (e.name, e.port), +) + + +NEGATIVE: List[Endpoint] = [ + Endpoint("169.254.169.254", 80), + Endpoint("metadata.google.internal", 80), +] diff --git a/stack-compose.yml b/stack-compose.yml new file mode 100644 index 0000000..ece5023 --- /dev/null +++ b/stack-compose.yml @@ -0,0 +1,28 @@ +services: + connectivity-tester: + build: + context: . + dockerfile: Dockerfile + image: gitea.rudolf.keenetic.link/Org1/swarm_connectivity_tester:${version} + hostname: connectivity-tester + user: "0:0" + volumes: + - /var/run/docker.sock:/var/run/docker.sock + healthcheck: + test: ["CMD-SHELL", "test -S /var/run/docker.sock"] + interval: 60s + timeout: 5s + retries: 3 + start_period: 30s + deploy: + mode: global + resources: + limits: + memory: 128M + pids: 64 + reservations: + memory: 64M + restart_policy: + condition: on-failure + delay: 10s + max_attempts: 3