forked from admin/ansible
feat(connectivity_tester): rotate networks per-round, cross-validate map
Self-attach/detach to overlay networks via docker socket. At any moment the container is on exactly one network; per-network list of expected services is loaded from network_map.NETWORK_MAP. After checks run cross-validation: - service reachable but not in map -> ERROR 'LEAK-UNEXPECTED' - service in map but unreachable -> ERROR 'MISSING-EXPECTED' - global service not in any map -> WARNING 'GLOBAL-UNCOVERED' Compose uses /var/run/docker.sock bind mount and removes overlay-network references; EXTRA_ENDPOINTS / HTTP_PROXY / HTTPS_PROXY / NO_PROXY are gone. endpoints.py removed (its data moved into network_map.py).
This commit is contained in:
@@ -3,6 +3,8 @@ FROM python:3.12-slim
|
||||
WORKDIR /app
|
||||
COPY . /app/
|
||||
|
||||
RUN pip install --no-cache-dir docker==7.1.0
|
||||
|
||||
ENV PYTHONUNBUFFERED=1 \
|
||||
PERIOD_SECONDS=600 \
|
||||
CHECK_TIMEOUT_S=3
|
||||
|
||||
+159
-58
@@ -1,11 +1,10 @@
|
||||
"""Connectivity tester: verifies internal/external endpoints are reachable
|
||||
and negative-list endpoints are not. Logs structured output, alerts on
|
||||
unexpected failures.
|
||||
"""Connectivity tester.
|
||||
|
||||
ENV:
|
||||
EXTRA_ENDPOINTS - comma-separated host:port pairs, ad-hoc checks
|
||||
PERIOD_SECONDS - sleep between full check rounds (default 600)
|
||||
CHECK_TIMEOUT_S - per-endpoint timeout in seconds (default 3)
|
||||
For every network in network_map.NETWORK_MAP the container attaches itself,
|
||||
runs TCP checks against the listed services, detaches, and proceeds to the
|
||||
next network. Cross-validates GLOBAL_SERVICES against each network map: logs
|
||||
errors when reachable services are absent from the map or when expected
|
||||
services are unreachable.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -15,30 +14,86 @@ import os
|
||||
import socket
|
||||
import sys
|
||||
import time
|
||||
from typing import Iterable, List, Tuple
|
||||
from typing import Iterable, List, Set, Tuple
|
||||
|
||||
from endpoints import EXPECTED_EXTERNAL, EXPECTED_INTERNAL, NEGATIVE, Endpoint
|
||||
import docker
|
||||
from docker.models.containers import Container
|
||||
from docker.models.networks import Network
|
||||
|
||||
from network_map import GLOBAL_SERVICES, NETWORK_MAP, Service
|
||||
|
||||
LOG = logging.getLogger("connectivity-tester")
|
||||
|
||||
|
||||
def _parse_extra(raw: str) -> List[Endpoint]:
|
||||
out: List[Endpoint] = []
|
||||
for token in raw.split(","):
|
||||
token = token.strip()
|
||||
if not token:
|
||||
def _socket_path() -> str:
|
||||
return os.environ.get("DOCKER_SOCKET", "/var/run/docker.sock")
|
||||
|
||||
|
||||
def _self_container(client: docker.DockerClient) -> Container:
|
||||
hostname = socket.gethostname()
|
||||
for c in client.containers.list():
|
||||
if (c.attrs.get("Config", {}).get("Hostname") == hostname
|
||||
or c.name.lstrip("/") == hostname):
|
||||
return c
|
||||
candidates = [c for c in client.containers.list()
|
||||
if hostname[:12] in c.id]
|
||||
if len(candidates) == 1:
|
||||
return candidates[0]
|
||||
raise RuntimeError(f"self container not found for hostname={hostname}")
|
||||
|
||||
|
||||
def _filtered_networks(client: docker.DockerClient) -> List[Network]:
|
||||
keep_prefixes = (
|
||||
"hermes_infra_", "gitea_", "gmail_mcp_", "telegram_",
|
||||
"hermes_", "brencher", "registry_", "immich_",
|
||||
"torrserv_proxy_", "photosHelper_", "nginx_",
|
||||
"docker_socket_proxy_",
|
||||
)
|
||||
out = []
|
||||
for n in client.networks.list():
|
||||
if n.attrs.get("Scope") != "swarm":
|
||||
continue
|
||||
if ":" not in token:
|
||||
name = n.name
|
||||
if name in ("ingress",):
|
||||
continue
|
||||
host, port_s = token.rsplit(":", 1)
|
||||
try:
|
||||
port = int(port_s)
|
||||
except ValueError:
|
||||
continue
|
||||
out.append((host, port, "extra"))
|
||||
if any(name.startswith(p) for p in keep_prefixes):
|
||||
out.append(n)
|
||||
return out
|
||||
|
||||
|
||||
def _build_map_from_running(networks: Iterable[Network]) -> dict:
|
||||
"""Reverse-engineer NAME -> swarm network name from running container endpoints."""
|
||||
out = {}
|
||||
for net in networks:
|
||||
endpoint_count = len(net.attrs.get("Containers") or {})
|
||||
out[net.name] = {"endpoints": endpoint_count, "id": net.id}
|
||||
return out
|
||||
|
||||
|
||||
def _attach(client: docker.DockerClient, container: Container,
|
||||
network: Network) -> None:
|
||||
if network.name in (container.attrs.get("NetworkSettings", {})
|
||||
.get("Networks", {}).keys()):
|
||||
return
|
||||
try:
|
||||
network.connect(container, aliases=["connectivity-tester",
|
||||
socket.gethostname()])
|
||||
except Exception as exc:
|
||||
if "already exists" in str(exc).lower():
|
||||
return
|
||||
raise
|
||||
|
||||
|
||||
def _detach(client: docker.DockerClient, container: Container,
|
||||
network: Network) -> None:
|
||||
try:
|
||||
network.disconnect(container, force=True)
|
||||
except Exception as exc:
|
||||
if "not found" in str(exc).lower():
|
||||
return
|
||||
raise
|
||||
|
||||
|
||||
async def _tcp_check(host: str, port: int, timeout: float) -> Tuple[bool, str]:
|
||||
try:
|
||||
fut = asyncio.open_connection(host, port)
|
||||
@@ -55,45 +110,60 @@ async def _tcp_check(host: str, port: int, timeout: float) -> Tuple[bool, str]:
|
||||
return False, f"{type(exc).__name__}: {exc}"
|
||||
|
||||
|
||||
async def _check_positive(endpoints: Iterable[Endpoint], timeout: float) -> int:
|
||||
failures = 0
|
||||
for host, port, label in endpoints:
|
||||
ok, detail = await _tcp_check(host, port, timeout)
|
||||
if ok:
|
||||
LOG.info("PASS %s:%d (%s) %s", host, port, label, detail)
|
||||
else:
|
||||
LOG.error("FAIL %s:%d (%s) %s", host, port, label, detail)
|
||||
failures += 1
|
||||
return failures
|
||||
async def _check_one(host: str, port: int, label: str,
|
||||
timeout: float) -> bool:
|
||||
ok, detail = await _tcp_check(host, port, timeout)
|
||||
if ok:
|
||||
LOG.info("PASS %s:%d (%s) %s", host, port, label, detail)
|
||||
else:
|
||||
LOG.error("FAIL %s:%d (%s) %s", host, port, label, detail)
|
||||
return ok
|
||||
|
||||
|
||||
async def _check_negative(endpoints: Iterable[Endpoint], timeout: float) -> int:
|
||||
leaks = 0
|
||||
for host, port, label in endpoints:
|
||||
ok, detail = await _tcp_check(host, port, timeout)
|
||||
if ok:
|
||||
LOG.error(
|
||||
"LEAK %s:%d (%s) reachable but must NOT be: %s",
|
||||
host, port, label, detail,
|
||||
)
|
||||
leaks += 1
|
||||
else:
|
||||
LOG.info("OK-NEG %s:%d (%s) blocked: %s", host, port, label, detail)
|
||||
return leaks
|
||||
async def _check_network(network: Network, services: List[Service],
|
||||
timeout: float) -> Tuple[Set[Service], Set[Service]]:
|
||||
reachable: Set[Service] = set()
|
||||
unreachable: Set[Service] = set()
|
||||
for host, port in services:
|
||||
ok = await _check_one(host, port, network.name, timeout)
|
||||
(reachable if ok else unreachable).add((host, port))
|
||||
return reachable, unreachable
|
||||
|
||||
|
||||
async def _round(timeout: float) -> int:
|
||||
extras = _parse_extra(os.environ.get("EXTRA_ENDPOINTS", ""))
|
||||
LOG.info("--- round begin, %d internal + %d external + %d extra + %d negative ---",
|
||||
len(EXPECTED_INTERNAL), len(EXPECTED_EXTERNAL), len(extras), len(NEGATIVE))
|
||||
failures = 0
|
||||
failures += await _check_positive(EXPECTED_INTERNAL, timeout)
|
||||
failures += await _check_positive(EXPECTED_EXTERNAL, timeout)
|
||||
failures += await _check_positive(extras, timeout)
|
||||
leaks = await _check_negative(NEGATIVE, timeout)
|
||||
failures += leaks
|
||||
LOG.info("--- round end, %d failure(s) ---", failures)
|
||||
return failures
|
||||
def _discover_attached(client: docker.DockerClient,
|
||||
container: Container) -> Set[str]:
|
||||
return set((container.attrs.get("NetworkSettings", {}) or {})
|
||||
.get("Networks", {}).keys())
|
||||
|
||||
|
||||
def _cross_validate(network_name: str,
|
||||
mapped: List[Service],
|
||||
reachable: Set[Service],
|
||||
unreachable: Set[Service],
|
||||
all_global: List[Service]) -> None:
|
||||
mapped_set = set(mapped)
|
||||
reachable_global = reachable & set(all_global)
|
||||
for svc in reachable_global - mapped_set:
|
||||
LOG.error("LEAK-UNEXPECTED %s reachable in %s but not in map",
|
||||
f"{svc[0]}:{svc[1]}", network_name)
|
||||
for svc in mapped_set & unreachable:
|
||||
LOG.error("MISSING-EXPECTED %s unreachable in %s but listed in map",
|
||||
f"{svc[0]}:{svc[1]}", network_name)
|
||||
for svc in mapped_set & reachable:
|
||||
LOG.info("OK-MAPPED %s ok in %s", f"{svc[0]}:{svc[1]}", network_name)
|
||||
for svc in mapped_set - reachable - unreachable:
|
||||
LOG.warning("NOT-CHECKED %s in map for %s",
|
||||
f"{svc[0]}:{svc[1]}", network_name)
|
||||
|
||||
|
||||
def _consistency_audit(networks_names: List[str], all_global: List[Service]) -> None:
|
||||
flat: Set[Service] = set()
|
||||
for n in networks_names:
|
||||
flat.update(NETWORK_MAP.get(n, []))
|
||||
leftover = set(all_global) - flat
|
||||
if leftover:
|
||||
LOG.warning("GLOBAL-UNCOVERED %s not present in any map",
|
||||
sorted(f"{s[0]}:{s[1]}" for s in leftover))
|
||||
|
||||
|
||||
async def main() -> int:
|
||||
@@ -104,13 +174,44 @@ async def main() -> int:
|
||||
)
|
||||
period = int(os.environ.get("PERIOD_SECONDS", "600"))
|
||||
timeout = float(os.environ.get("CHECK_TIMEOUT_S", "3"))
|
||||
LOG.info("starting; period=%ds, check_timeout=%.1fs", period, timeout)
|
||||
LOG.info("starting; period=%ds timeout=%.1fs", period, timeout)
|
||||
|
||||
client = docker.DockerClient(base_url="unix://" + _socket_path())
|
||||
self_container = _self_container(client)
|
||||
self_id_short = self_container.id[:12]
|
||||
|
||||
networks_docker = _filtered_networks(client)
|
||||
networks_known = [n for n in networks_docker if n.name in NETWORK_MAP]
|
||||
LOG.info("configured networks: %d, present in swarm: %d",
|
||||
len(NETWORK_MAP), len(networks_known))
|
||||
for n in networks_known:
|
||||
LOG.info("plan network: %s", n.name)
|
||||
|
||||
_consistency_audit([n.name for n in networks_known], GLOBAL_SERVICES)
|
||||
|
||||
while True:
|
||||
started = time.monotonic()
|
||||
failures = 0
|
||||
try:
|
||||
await _round(timeout)
|
||||
for network in networks_known:
|
||||
LOG.info("--- round begin, attaching to %s ---", network.name)
|
||||
_attach(client, self_container, network)
|
||||
try:
|
||||
reachable, unreachable = await _check_network(
|
||||
network, NETWORK_MAP[network.name], timeout)
|
||||
_cross_validate(
|
||||
network.name,
|
||||
NETWORK_MAP[network.name],
|
||||
reachable,
|
||||
unreachable,
|
||||
GLOBAL_SERVICES,
|
||||
)
|
||||
finally:
|
||||
_detach(client, self_container, network)
|
||||
LOG.info("--- round end on %s ---", network.name)
|
||||
except Exception:
|
||||
LOG.exception("round crashed")
|
||||
|
||||
elapsed = time.monotonic() - started
|
||||
sleep_for = max(0.0, period - elapsed)
|
||||
await asyncio.sleep(sleep_for)
|
||||
|
||||
@@ -1,46 +0,0 @@
|
||||
"""Endpoint allow-list and negative-list for connectivity-tester.
|
||||
|
||||
EXPECTED_INTERNAL: TCP services in overlay networks that MUST be reachable.
|
||||
EXPECTED_EXTERNAL: external hosts that MUST be reachable via squid egress.
|
||||
NEGATIVE: hosts that MUST NOT respond (proxy / firewall sanity check).
|
||||
|
||||
Each entry is (host, port, label).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import List, Tuple
|
||||
|
||||
Endpoint = Tuple[str, int, str]
|
||||
|
||||
|
||||
EXPECTED_INTERNAL: List[Endpoint] = [
|
||||
("squid", 3128, "egress proxy"),
|
||||
("hermes-gateway", 8000, "hermes messaging gateway"),
|
||||
("hermes-ui", 8087, "hermes dashboard ui"),
|
||||
("docker_socket_proxy", 2375, "read-only docker socket proxy"),
|
||||
("brencher-state-proxy", 80, "brencher state reverse proxy"),
|
||||
("gitea-server", 3000, "gitea http"),
|
||||
("act_runner", 8088, "gitea actions runner"),
|
||||
("telegram-mcp-server", 8000, "telegram mcp"),
|
||||
("gmail-mcp-server", 8000, "gmail mcp"),
|
||||
("voice-gateway", 8765, "voice realtime ws bridge"),
|
||||
("hermes-proxy", 8645, "openai-compatible http egress"),
|
||||
]
|
||||
|
||||
|
||||
EXPECTED_EXTERNAL: List[Endpoint] = [
|
||||
("huggingface.co", 443, "piper voice model download"),
|
||||
("registry.rudolf.keenetic.link", 5000, "local docker registry"),
|
||||
("gitea.rudolf.keenetic.link", 443, "public gitea"),
|
||||
("pypi.org", 443, "python package index"),
|
||||
("github.com", 443, "github api"),
|
||||
("google.com", 443, "egress sanity check"),
|
||||
("192.168.1.1", 80, "lan gateway .1"),
|
||||
("192.168.0.1", 80, "lan gateway .0"),
|
||||
]
|
||||
|
||||
|
||||
NEGATIVE: List[Endpoint] = [
|
||||
("169.254.169.254", 80, "cloud metadata service"),
|
||||
("metadata.google.internal", 80, "gcp metadata"),
|
||||
]
|
||||
@@ -0,0 +1,40 @@
|
||||
from typing import Dict, List, Tuple
|
||||
|
||||
Service = Tuple[str, int]
|
||||
|
||||
GLOBAL_SERVICES: List[Service] = [
|
||||
("hermes-gateway", 8000),
|
||||
("hermes-agent", 8000),
|
||||
("hermes-ui", 8087),
|
||||
("squid", 3128),
|
||||
("gitea-server", 3000),
|
||||
("act_runner", 8088),
|
||||
("brencher-state-proxy", 80),
|
||||
("docker_socket_proxy", 2375),
|
||||
("telegram-mcp-server", 8000),
|
||||
("gmail-mcp-server", 8000),
|
||||
("voice-gateway", 8765),
|
||||
("hermes-proxy", 8645),
|
||||
]
|
||||
|
||||
NETWORK_MAP: Dict[str, List[Service]] = {
|
||||
"hermes_infra_hermes_proxy": [
|
||||
("hermes-gateway", 8000),
|
||||
("hermes-agent", 8000),
|
||||
("hermes-ui", 8087),
|
||||
("squid", 3128),
|
||||
("gitea-server", 3000),
|
||||
("act_runner", 8088),
|
||||
("brencher-state-proxy", 80),
|
||||
("docker_socket_proxy", 2375),
|
||||
("telegram-mcp-server", 8000),
|
||||
("gmail-mcp-server", 8000),
|
||||
("voice-gateway", 8765),
|
||||
("hermes-proxy", 8645),
|
||||
],
|
||||
"gitea_gitea": [
|
||||
("gitea-server", 3000),
|
||||
("act_runner", 8088),
|
||||
("cache_server", 37051),
|
||||
],
|
||||
}
|
||||
@@ -6,18 +6,10 @@ services:
|
||||
image: registry.rudolf.keenetic.link/connectivity_tester:${version}
|
||||
hostname: connectivity-tester
|
||||
user: "0:0"
|
||||
networks:
|
||||
- hermes_proxy
|
||||
- default
|
||||
environment:
|
||||
PERIOD_SECONDS: "600"
|
||||
CHECK_TIMEOUT_S: "3"
|
||||
EXTRA_ENDPOINTS: ""
|
||||
HTTP_PROXY: "http://squid:3128"
|
||||
HTTPS_PROXY: "http://squid:3128"
|
||||
NO_PROXY: "localhost,127.0.0.1,.local,squid,hermes-gateway,hermes-agent,hermes-proxy,brencher-state-proxy,docker_socket_proxy,10.0.0.0/8"
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "python -c \"import socket; s=socket.socket(); s.settimeout(2); s.connect(('squid', 3128)); s.close()\""]
|
||||
test: ["CMD-SHELL", "test -S /var/run/docker.sock"]
|
||||
interval: 60s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
@@ -34,9 +26,3 @@ services:
|
||||
condition: on-failure
|
||||
delay: 10s
|
||||
max_attempts: 3
|
||||
|
||||
networks:
|
||||
hermes_proxy:
|
||||
external: true
|
||||
default:
|
||||
external: true
|
||||
|
||||
Reference in New Issue
Block a user