Checks node-local prerequisites that break Hermes after relocation: direct TLS egress to api.telegram.org and real NFS read/write against the Hermes agent export. Removes the tester self-probe that caused every round to crash. Tests: compileall, unittest, git diff --check. Reviewed-on: admin/ansible#50 Co-authored-by: Alex Rudenk <rudolfss13+hermes@gmail.com>
361 lines
13 KiB
Python
361 lines
13 KiB
Python
"""Connectivity tester.
|
|
|
|
For every configured overlay network the container attaches itself, runs
|
|
TCP checks against its endpoints, then detaches. Endpoints marked with
|
|
``proxy=(host, port)`` are probed through that HTTP proxy (CONNECT). After
|
|
each round a final loop checks PUBLIC_SERVICES. Cross-validation:
|
|
- service reachable but not in map -> ERROR 'LEAK-UNEXPECTED'
|
|
- service in map but unreachable -> ERROR 'MISSING-EXPECTED'
|
|
- global service not in any map -> WARNING 'GLOBAL-UNCOVERED'
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
import os
|
|
import ssl
|
|
import socket
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
from typing import Iterable, List, Set, Tuple
|
|
|
|
import docker
|
|
from docker.models.containers import Container
|
|
from docker.models.networks import Network
|
|
|
|
from network_map import Endpoint, GLOBAL_SERVICES, NEGATIVE, NETWORK_MAP, PUBLIC_SERVICES
|
|
from host_checks import build_host_checks, parse_peers, tcp_probe, udp_probe
|
|
|
|
LOG = logging.getLogger("connectivity-tester")
|
|
|
|
FALLBACK_TCP_TIMEOUT_S = 3.0
|
|
HTTP_CONNECT_TIMEOUT_S = 5.0
|
|
HOST_TCP_PORTS = (2377, 7946, 2049)
|
|
HOST_UDP_PORTS = (7946, 4789)
|
|
DIRECT_INTERNET_HOST = "api.telegram.org"
|
|
DIRECT_INTERNET_PORT = 443
|
|
NFS_PROBE_PATH = os.environ.get("NFS_PROBE_PATH", "/mnt/hermes-nfs-probe")
|
|
|
|
|
|
def _socket_path() -> str:
|
|
return os.environ.get("DOCKER_SOCKET", "/var/run/docker.sock")
|
|
|
|
|
|
def _self_container(client: docker.DockerClient) -> Container:
|
|
hostname = socket.gethostname()
|
|
for c in client.containers.list():
|
|
cn = c.name.lstrip("/")
|
|
hn = c.attrs.get("Config", {}).get("Hostname", "")
|
|
if hostname == cn or hostname == hn or hostname[:12] == c.id[:12]:
|
|
return c
|
|
candidates = [c for c in client.containers.list() if hostname[:12] == c.id]
|
|
if len(candidates) == 1:
|
|
return candidates[0]
|
|
raise RuntimeError(f"self container not found for hostname={hostname}")
|
|
|
|
|
|
def _swarm_networks_for(client: docker.DockerClient,
|
|
names: Iterable[str]) -> dict:
|
|
out = {}
|
|
by_name = {n.name: n for n in client.networks.list()}
|
|
for name in names:
|
|
net = by_name.get(name)
|
|
if net is not None and net.attrs.get("Scope") == "swarm":
|
|
out[name] = net
|
|
return out
|
|
|
|
|
|
def _attach(client: docker.DockerClient, container: Container,
|
|
network: Network) -> bool:
|
|
current = (container.attrs.get("NetworkSettings", {}) or {}).get("Networks", {})
|
|
if network.name in current:
|
|
return False
|
|
try:
|
|
network.connect(container, aliases=["connectivity-tester",
|
|
socket.gethostname()])
|
|
return True
|
|
except Exception as exc:
|
|
if "already exists" in str(exc).lower():
|
|
return False
|
|
raise
|
|
|
|
|
|
def _detach(client: docker.DockerClient, container: Container,
|
|
network: Network) -> None:
|
|
try:
|
|
network.disconnect(container, force=True)
|
|
except Exception as exc:
|
|
if "not found" in str(exc).lower():
|
|
return
|
|
raise
|
|
|
|
|
|
async def _tcp_probe(host: str, port: int, timeout: float) -> Tuple[bool, str]:
|
|
try:
|
|
fut = asyncio.open_connection(host=host, port=port)
|
|
reader, writer = await asyncio.wait_for(fut, timeout=timeout)
|
|
writer.close()
|
|
try:
|
|
await writer.wait_closed()
|
|
except Exception:
|
|
pass
|
|
return True, "tcp ok"
|
|
except asyncio.TimeoutError:
|
|
return False, "timeout"
|
|
except OSError as exc:
|
|
return False, f"{type(exc).__name__}: {exc}"
|
|
|
|
|
|
async def _tls_probe(host: str, port: int, timeout: float) -> Tuple[bool, str]:
|
|
context = ssl.create_default_context()
|
|
try:
|
|
_, writer = await asyncio.wait_for(
|
|
asyncio.open_connection(host, port, ssl=context, server_hostname=host),
|
|
timeout=timeout,
|
|
)
|
|
writer.close()
|
|
try:
|
|
await writer.wait_closed()
|
|
except Exception:
|
|
pass
|
|
return True, "tls handshake ok"
|
|
except asyncio.TimeoutError:
|
|
return False, "timeout"
|
|
except (OSError, ssl.SSLError) as exc:
|
|
return False, f"{type(exc).__name__}: {exc}"
|
|
|
|
|
|
async def _proxy_connect(endpoint: Endpoint, timeout: float) -> Tuple[bool, str]:
|
|
if endpoint.proxy is None:
|
|
return await _tcp_probe(endpoint.name, endpoint.port, timeout=timeout)
|
|
proxy_host, proxy_port = endpoint.proxy
|
|
target = endpoint.name
|
|
target_port = endpoint.port
|
|
try:
|
|
fut = asyncio.open_connection(host=proxy_host, port=proxy_port)
|
|
reader, writer = await asyncio.wait_for(fut, timeout=timeout)
|
|
req = (
|
|
f"CONNECT {target}:{target_port} HTTP/1.1\r\n"
|
|
f"Host: {target}:{target_port}\r\n\r\n"
|
|
)
|
|
writer.write(req.encode("ascii"))
|
|
await writer.drain()
|
|
status_line = await asyncio.wait_for(reader.readline(),
|
|
timeout=timeout)
|
|
if not status_line.startswith(b"HTTP/") or b" 200 " not in status_line:
|
|
writer.close()
|
|
try:
|
|
await writer.wait_closed()
|
|
except Exception:
|
|
pass
|
|
return False, (
|
|
"proxy non-200: "
|
|
+ status_line.strip().decode(errors="replace")
|
|
)
|
|
while True:
|
|
line = await asyncio.wait_for(reader.readline(), timeout=timeout)
|
|
if line in (b"\r\n", b"", b"\n"):
|
|
break
|
|
writer.close()
|
|
try:
|
|
await writer.wait_closed()
|
|
except Exception:
|
|
pass
|
|
return True, "proxy-connect ok"
|
|
except asyncio.TimeoutError:
|
|
return False, "timeout"
|
|
except OSError as exc:
|
|
return False, f"{type(exc).__name__}: {exc}"
|
|
|
|
|
|
async def _check_endpoint(endpoint: Endpoint, timeout: float) -> bool:
|
|
if endpoint.proxy is not None:
|
|
ok, detail = await _proxy_connect(endpoint, timeout)
|
|
else:
|
|
ok, detail = await _tcp_probe(endpoint.name, endpoint.port,
|
|
timeout=timeout)
|
|
label = f"{endpoint.name}:{endpoint.port}"
|
|
src = f"via {endpoint.proxy}" if endpoint.proxy else "direct"
|
|
if ok:
|
|
LOG.info("PASS %s %s %s", label, src, detail)
|
|
else:
|
|
LOG.error("FAIL %s %s %s", label, src, detail)
|
|
return ok
|
|
|
|
|
|
async def _check_network(network_name: str,
|
|
endpoints: List[Endpoint],
|
|
timeout: float) -> Tuple[Set[Endpoint], Set[Endpoint]]:
|
|
reachable: Set[Endpoint] = set()
|
|
unreachable: Set[Endpoint] = set()
|
|
for ep in endpoints:
|
|
ok = await _check_endpoint(ep, timeout)
|
|
(reachable if ok else unreachable).add(ep)
|
|
return reachable, unreachable
|
|
|
|
|
|
def _cross_validate(network_name: str,
|
|
mapped: List[Endpoint],
|
|
reachable: Set[Endpoint],
|
|
unreachable: Set[Endpoint],
|
|
global_set: Set[Endpoint]) -> None:
|
|
mapped_set = set(mapped)
|
|
reachable_global = reachable & global_set
|
|
for ep in reachable_global - mapped_set:
|
|
LOG.error("LEAK-UNEXPECTED %s reachable in %s but not in map",
|
|
f"{ep.name}:{ep.port}", network_name)
|
|
for ep in mapped_set & unreachable:
|
|
LOG.error("MISSING-EXPECTED %s unreachable in %s but listed in map",
|
|
f"{ep.name}:{ep.port}", network_name)
|
|
for ep in mapped_set & reachable:
|
|
LOG.info("OK-MAPPED %s ok in %s", f"{ep.name}:{ep.port}", network_name)
|
|
for ep in mapped_set - reachable - unreachable:
|
|
LOG.warning("NOT-CHECKED %s in map for %s",
|
|
f"{ep.name}:{ep.port}", network_name)
|
|
|
|
|
|
def _consistency_audit(net_names: List[str]) -> None:
|
|
flat: Set[Endpoint] = set()
|
|
for n in net_names:
|
|
flat.update(NETWORK_MAP.get(n, []))
|
|
covered = flat | set(PUBLIC_SERVICES)
|
|
all_global = set(GLOBAL_SERVICES)
|
|
uncovered = all_global - covered
|
|
if uncovered:
|
|
LOG.warning("GLOBAL-UNCOVERED %s not present in any map",
|
|
sorted(f"{e.name}:{e.port}" for e in uncovered))
|
|
|
|
|
|
async def _check_negative(endpoints: List[Endpoint],
|
|
timeout: float) -> int:
|
|
leaks = 0
|
|
for ep in endpoints:
|
|
ok = await _tcp_probe(ep.name, ep.port, timeout=timeout)
|
|
if ok:
|
|
LOG.error("LEAK %s:%d reachable but must NOT be",
|
|
ep.name, ep.port)
|
|
leaks += 1
|
|
else:
|
|
LOG.info("OK-NEG %s:%d blocked", ep.name, ep.port)
|
|
return leaks
|
|
|
|
|
|
async def _check_host_connectivity(
|
|
peers: List[Tuple[str, str]], timeout: float
|
|
) -> None:
|
|
checks = build_host_checks(peers, HOST_TCP_PORTS, HOST_UDP_PORTS)
|
|
LOG.info("--- host connectivity checks (%d probes) ---", len(checks))
|
|
for name, address, protocol, port in checks:
|
|
if protocol == "tcp":
|
|
ok, detail = await tcp_probe(address, port, timeout)
|
|
else:
|
|
ok, detail = await udp_probe(address, port, timeout)
|
|
label = f"{name} {address}:{port}/{protocol}"
|
|
if ok:
|
|
LOG.info("PASS-HOST %s %s", label, detail)
|
|
else:
|
|
LOG.error("FAIL-HOST %s %s", label, detail)
|
|
|
|
ok, detail = await _tls_probe(
|
|
DIRECT_INTERNET_HOST,
|
|
DIRECT_INTERNET_PORT,
|
|
max(timeout, HTTP_CONNECT_TIMEOUT_S),
|
|
)
|
|
label = f"{DIRECT_INTERNET_HOST}:{DIRECT_INTERNET_PORT}"
|
|
if ok:
|
|
LOG.info("PASS-HOST %s direct %s", label, detail)
|
|
else:
|
|
LOG.error("FAIL-HOST %s direct %s", label, detail)
|
|
|
|
|
|
def _check_nfs() -> bool:
|
|
path = Path(NFS_PROBE_PATH)
|
|
probe = path / (
|
|
f".connectivity-tester-{socket.gethostname()}-"
|
|
f"{os.getpid()}-{time.time_ns()}"
|
|
)
|
|
payload = f"{socket.gethostname()}:{time.time_ns()}\n"
|
|
started = time.monotonic()
|
|
try:
|
|
probe.write_text(payload, encoding="utf-8")
|
|
if probe.read_text(encoding="utf-8") != payload:
|
|
raise OSError("read-back mismatch")
|
|
LOG.info("PASS-HOST nfs %s read/write latency=%.3fs",
|
|
NFS_PROBE_PATH, time.monotonic() - started)
|
|
return True
|
|
except OSError as exc:
|
|
LOG.error("FAIL-HOST nfs %s %s", NFS_PROBE_PATH, exc)
|
|
return False
|
|
finally:
|
|
try:
|
|
probe.unlink()
|
|
except FileNotFoundError:
|
|
pass
|
|
|
|
|
|
async def main() -> int:
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format="%(asctime)s %(levelname)s %(message)s",
|
|
stream=sys.stdout,
|
|
)
|
|
period = int(os.environ.get("PERIOD_SECONDS", "600"))
|
|
timeout = float(os.environ.get("CHECK_TIMEOUT_S", "3"))
|
|
peers = parse_peers(os.environ.get("SWARM_PEERS", ""))
|
|
LOG.info("starting; period=%ds timeout=%.1fs", period, timeout)
|
|
LOG.info("configured swarm peers: %d", len(peers))
|
|
|
|
client = docker.DockerClient(base_url="unix://" + _socket_path())
|
|
self_container = _self_container(client)
|
|
|
|
nets = _swarm_networks_for(client, NETWORK_MAP.keys())
|
|
planned = list(NETWORK_MAP.keys())
|
|
LOG.info("configured networks: %d, present in swarm: %d",
|
|
len(planned), len(nets))
|
|
for n in nets:
|
|
LOG.info("plan network: %s", n)
|
|
|
|
_consistency_audit(planned)
|
|
|
|
global_set: Set[Endpoint] = set(GLOBAL_SERVICES)
|
|
|
|
while True:
|
|
started = time.monotonic()
|
|
try:
|
|
for net_name, net in nets.items():
|
|
eps_in_map: List[Endpoint] = [
|
|
e for e in NETWORK_MAP[net_name] if e.proxy is None
|
|
]
|
|
LOG.info("--- round begin, attaching to %s (%d endpoints) ---",
|
|
net_name, len(eps_in_map))
|
|
_attach(client, self_container, net)
|
|
try:
|
|
reachable, unreachable = await _check_network(
|
|
net_name, eps_in_map, timeout)
|
|
_cross_validate(net_name, eps_in_map,
|
|
reachable, unreachable, global_set)
|
|
finally:
|
|
_detach(client, self_container, net)
|
|
LOG.info("--- round end on %s ---", net_name)
|
|
|
|
LOG.info("--- public checks (%d endpoints) ---",
|
|
len(PUBLIC_SERVICES))
|
|
for ep in PUBLIC_SERVICES:
|
|
await _check_endpoint(ep, max(timeout, HTTP_CONNECT_TIMEOUT_S))
|
|
|
|
LOG.info("--- negative checks (%d endpoints) ---", len(NEGATIVE))
|
|
await _check_negative(NEGATIVE, timeout)
|
|
await _check_host_connectivity(peers, timeout)
|
|
_check_nfs()
|
|
except Exception:
|
|
LOG.exception("round crashed")
|
|
|
|
elapsed = time.monotonic() - started
|
|
sleep_for = max(0.0, period - elapsed)
|
|
await asyncio.sleep(sleep_for)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main())
|