forked from admin/ansible
Replace tuple-of-strings with frozen Endpoint dataclass carrying optional (proxy_host, proxy_port) tuple. Add network_map.NETWORK_MAP populated from the live swarm endpoints discovered via the docker socket, and a separate PUBLIC_SERVICES list sourced from the squid allowlist plus well-known public hosts. Endpoints with proxy are checked via HTTP CONNECT through that proxy; endpoints without are probed directly via TCP after attaching to the relevant overlay network. Cross-validation rules cover leaks and missing expectations per round.
277 lines
9.7 KiB
Python
277 lines
9.7 KiB
Python
"""Connectivity tester.
|
|
|
|
For every configured overlay network the container attaches itself, runs
|
|
TCP checks against its endpoints, then detaches. Endpoints marked with
|
|
``proxy=(host, port)`` are probed through that HTTP proxy (CONNECT). After
|
|
each round a final loop checks PUBLIC_SERVICES. Cross-validation:
|
|
- service reachable but not in map -> ERROR 'LEAK-UNEXPECTED'
|
|
- service in map but unreachable -> ERROR 'MISSING-EXPECTED'
|
|
- global service not in any map -> WARNING 'GLOBAL-UNCOVERED'
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
import os
|
|
import socket
|
|
import sys
|
|
import time
|
|
from typing import Iterable, List, Set, Tuple
|
|
|
|
import docker
|
|
from docker.models.containers import Container
|
|
from docker.models.networks import Network
|
|
|
|
from network_map import Endpoint, GLOBAL_SERVICES, NEGATIVE, NETWORK_MAP, PUBLIC_SERVICES
|
|
|
|
LOG = logging.getLogger("connectivity-tester")
|
|
|
|
FALLBACK_TCP_TIMEOUT_S = 3.0
|
|
HTTP_CONNECT_TIMEOUT_S = 5.0
|
|
|
|
|
|
def _socket_path() -> str:
|
|
return os.environ.get("DOCKER_SOCKET", "/var/run/docker.sock")
|
|
|
|
|
|
def _self_container(client: docker.DockerClient) -> Container:
|
|
hostname = socket.gethostname()
|
|
for c in client.containers.list():
|
|
cn = c.name.lstrip("/")
|
|
hn = c.attrs.get("Config", {}).get("Hostname", "")
|
|
if hostname == cn or hostname == hn or hostname[:12] == c.id[:12]:
|
|
return c
|
|
candidates = [c for c in client.containers.list() if hostname[:12] == c.id]
|
|
if len(candidates) == 1:
|
|
return candidates[0]
|
|
raise RuntimeError(f"self container not found for hostname={hostname}")
|
|
|
|
|
|
def _swarm_networks_for(client: docker.DockerClient,
|
|
names: Iterable[str]) -> dict:
|
|
out = {}
|
|
by_name = {n.name: n for n in client.networks.list()}
|
|
for name in names:
|
|
net = by_name.get(name)
|
|
if net is not None and net.attrs.get("Scope") == "swarm":
|
|
out[name] = net
|
|
return out
|
|
|
|
|
|
def _attach(client: docker.DockerClient, container: Container,
|
|
network: Network) -> bool:
|
|
current = (container.attrs.get("NetworkSettings", {}) or {}).get("Networks", {})
|
|
if network.name in current:
|
|
return False
|
|
try:
|
|
network.connect(container, aliases=["connectivity-tester",
|
|
socket.gethostname()])
|
|
return True
|
|
except Exception as exc:
|
|
if "already exists" in str(exc).lower():
|
|
return False
|
|
raise
|
|
|
|
|
|
def _detach(client: docker.DockerClient, container: Container,
|
|
network: Network) -> None:
|
|
try:
|
|
network.disconnect(container, force=True)
|
|
except Exception as exc:
|
|
if "not found" in str(exc).lower():
|
|
return
|
|
raise
|
|
|
|
|
|
async def _tcp_probe(host: str, port: int, timeout: float) -> Tuple[bool, str]:
|
|
try:
|
|
fut = asyncio.open_connection(host=host, port=port)
|
|
reader, writer = await asyncio.wait_for(fut, timeout=timeout)
|
|
writer.close()
|
|
try:
|
|
await writer.wait_closed()
|
|
except Exception:
|
|
pass
|
|
return True, "tcp ok"
|
|
except asyncio.TimeoutError:
|
|
return False, "timeout"
|
|
except OSError as exc:
|
|
return False, f"{type(exc).__name__}: {exc}"
|
|
|
|
|
|
async def _proxy_connect(endpoint: Endpoint, timeout: float) -> Tuple[bool, str]:
|
|
if endpoint.proxy is None:
|
|
return await _tcp_probe(endpoint.name, endpoint.port, timeout=timeout)
|
|
proxy_host, proxy_port = endpoint.proxy
|
|
target = endpoint.name
|
|
target_port = endpoint.port
|
|
try:
|
|
fut = asyncio.open_connection(host=proxy_host, port=proxy_port)
|
|
reader, writer = await asyncio.wait_for(fut, timeout=timeout)
|
|
req = (
|
|
f"CONNECT {target}:{target_port} HTTP/1.1\r\n"
|
|
f"Host: {target}:{target_port}\r\n\r\n"
|
|
)
|
|
writer.write(req.encode("ascii"))
|
|
await writer.drain()
|
|
status_line = await asyncio.wait_for(reader.readline(),
|
|
timeout=timeout)
|
|
if not status_line.startswith(b"HTTP/") or b" 200 " not in status_line:
|
|
writer.close()
|
|
try:
|
|
await writer.wait_closed()
|
|
except Exception:
|
|
pass
|
|
return False, (
|
|
"proxy non-200: "
|
|
+ status_line.strip().decode(errors="replace")
|
|
)
|
|
while True:
|
|
line = await asyncio.wait_for(reader.readline(), timeout=timeout)
|
|
if line in (b"\r\n", b"", b"\n"):
|
|
break
|
|
writer.close()
|
|
try:
|
|
await writer.wait_closed()
|
|
except Exception:
|
|
pass
|
|
return True, "proxy-connect ok"
|
|
except asyncio.TimeoutError:
|
|
return False, "timeout"
|
|
except OSError as exc:
|
|
return False, f"{type(exc).__name__}: {exc}"
|
|
|
|
|
|
async def _check_endpoint(endpoint: Endpoint, timeout: float) -> bool:
|
|
if endpoint.proxy is not None:
|
|
ok, detail = await _proxy_connect(endpoint, timeout)
|
|
else:
|
|
ok, detail = await _tcp_probe(endpoint.name, endpoint.port,
|
|
timeout=timeout)
|
|
label = f"{endpoint.name}:{endpoint.port}"
|
|
src = f"via {endpoint.proxy}" if endpoint.proxy else "direct"
|
|
if ok:
|
|
LOG.info("PASS %s %s %s", label, src, detail)
|
|
else:
|
|
LOG.error("FAIL %s %s %s", label, src, detail)
|
|
return ok
|
|
|
|
|
|
async def _check_network(network_name: str,
|
|
endpoints: List[Endpoint],
|
|
timeout: float) -> Tuple[Set[Endpoint], Set[Endpoint]]:
|
|
reachable: Set[Endpoint] = set()
|
|
unreachable: Set[Endpoint] = set()
|
|
for ep in endpoints:
|
|
ok = await _check_endpoint(ep, timeout)
|
|
(reachable if ok else unreachable).add(ep)
|
|
return reachable, unreachable
|
|
|
|
|
|
def _cross_validate(network_name: str,
|
|
mapped: List[Endpoint],
|
|
reachable: Set[Endpoint],
|
|
unreachable: Set[Endpoint],
|
|
global_set: Set[Endpoint]) -> None:
|
|
mapped_set = set(mapped)
|
|
reachable_global = reachable & global_set
|
|
for ep in reachable_global - mapped_set:
|
|
LOG.error("LEAK-UNEXPECTED %s reachable in %s but not in map",
|
|
f"{ep.name}:{ep.port}", network_name)
|
|
for ep in mapped_set & unreachable:
|
|
LOG.error("MISSING-EXPECTED %s unreachable in %s but listed in map",
|
|
f"{ep.name}:{ep.port}", network_name)
|
|
for ep in mapped_set & reachable:
|
|
LOG.info("OK-MAPPED %s ok in %s", f"{ep.name}:{ep.port}", network_name)
|
|
for ep in mapped_set - reachable - unreachable:
|
|
LOG.warning("NOT-CHECKED %s in map for %s",
|
|
f"{ep.name}:{ep.port}", network_name)
|
|
|
|
|
|
def _consistency_audit(net_names: List[str]) -> None:
|
|
flat: Set[Endpoint] = set()
|
|
for n in net_names:
|
|
flat.update(NETWORK_MAP.get(n, []))
|
|
covered = flat | set(PUBLIC_SERVICES)
|
|
all_global = set(GLOBAL_SERVICES)
|
|
uncovered = all_global - covered
|
|
if uncovered:
|
|
LOG.warning("GLOBAL-UNCOVERED %s not present in any map",
|
|
sorted(f"{e.name}:{e.port}" for e in uncovered))
|
|
|
|
|
|
async def _check_negative(endpoints: List[Endpoint],
|
|
timeout: float) -> int:
|
|
leaks = 0
|
|
for ep in endpoints:
|
|
ok = await _tcp_probe(ep.name, ep.port, timeout=timeout)
|
|
if ok:
|
|
LOG.error("LEAK %s:%d reachable but must NOT be",
|
|
ep.name, ep.port)
|
|
leaks += 1
|
|
else:
|
|
LOG.info("OK-NEG %s:%d blocked", ep.name, ep.port)
|
|
return leaks
|
|
|
|
|
|
async def main() -> int:
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format="%(asctime)s %(levelname)s %(message)s",
|
|
stream=sys.stdout,
|
|
)
|
|
period = int(os.environ.get("PERIOD_SECONDS", "600"))
|
|
timeout = float(os.environ.get("CHECK_TIMEOUT_S", "3"))
|
|
LOG.info("starting; period=%ds timeout=%.1fs", period, timeout)
|
|
|
|
client = docker.DockerClient(base_url="unix://" + _socket_path())
|
|
self_container = _self_container(client)
|
|
|
|
nets = _swarm_networks_for(client, NETWORK_MAP.keys())
|
|
planned = list(NETWORK_MAP.keys())
|
|
LOG.info("configured networks: %d, present in swarm: %d",
|
|
len(planned), len(nets))
|
|
for n in nets:
|
|
LOG.info("plan network: %s", n)
|
|
|
|
_consistency_audit(planned)
|
|
|
|
global_set: Set[Endpoint] = set(GLOBAL_SERVICES)
|
|
|
|
while True:
|
|
started = time.monotonic()
|
|
try:
|
|
for net_name, net in nets.items():
|
|
eps_in_map: List[Endpoint] = [
|
|
e for e in NETWORK_MAP[net_name] if e.proxy is None
|
|
]
|
|
LOG.info("--- round begin, attaching to %s (%d endpoints) ---",
|
|
net_name, len(eps_in_map))
|
|
_attach(client, self_container, net)
|
|
try:
|
|
reachable, unreachable = await _check_network(
|
|
net_name, eps_in_map, timeout)
|
|
_cross_validate(net_name, eps_in_map,
|
|
reachable, unreachable, global_set)
|
|
finally:
|
|
_detach(client, self_container, net)
|
|
LOG.info("--- round end on %s ---", net_name)
|
|
|
|
LOG.info("--- public checks (%d endpoints) ---",
|
|
len(PUBLIC_SERVICES))
|
|
for ep in PUBLIC_SERVICES:
|
|
await _check_endpoint(ep, max(timeout, HTTP_CONNECT_TIMEOUT_S))
|
|
|
|
LOG.info("--- negative checks (%d endpoints) ---", len(NEGATIVE))
|
|
await _check_negative(NEGATIVE, timeout)
|
|
except Exception:
|
|
LOG.exception("round crashed")
|
|
|
|
elapsed = time.monotonic() - started
|
|
sleep_for = max(0.0, period - elapsed)
|
|
await asyncio.sleep(sleep_for)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main())
|