forked from admin/ansible
feat(connectivity): verify Hermes internet and NFS
This commit is contained in:
@@ -13,9 +13,11 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import ssl
|
||||
import socket
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Iterable, List, Set, Tuple
|
||||
|
||||
import docker
|
||||
@@ -23,11 +25,17 @@ from docker.models.containers import Container
|
||||
from docker.models.networks import Network
|
||||
|
||||
from network_map import Endpoint, GLOBAL_SERVICES, NEGATIVE, NETWORK_MAP, PUBLIC_SERVICES
|
||||
from host_checks import build_host_checks, parse_peers, tcp_probe, udp_probe
|
||||
|
||||
LOG = logging.getLogger("connectivity-tester")
|
||||
|
||||
FALLBACK_TCP_TIMEOUT_S = 3.0
|
||||
HTTP_CONNECT_TIMEOUT_S = 5.0
|
||||
HOST_TCP_PORTS = (2377, 7946, 2049)
|
||||
HOST_UDP_PORTS = (7946, 4789)
|
||||
DIRECT_INTERNET_HOST = "api.telegram.org"
|
||||
DIRECT_INTERNET_PORT = 443
|
||||
NFS_PROBE_PATH = os.environ.get("NFS_PROBE_PATH", "/mnt/hermes-nfs-probe")
|
||||
|
||||
|
||||
def _socket_path() -> str:
|
||||
@@ -99,6 +107,25 @@ async def _tcp_probe(host: str, port: int, timeout: float) -> Tuple[bool, str]:
|
||||
return False, f"{type(exc).__name__}: {exc}"
|
||||
|
||||
|
||||
async def _tls_probe(host: str, port: int, timeout: float) -> Tuple[bool, str]:
|
||||
context = ssl.create_default_context()
|
||||
try:
|
||||
_, writer = await asyncio.wait_for(
|
||||
asyncio.open_connection(host, port, ssl=context, server_hostname=host),
|
||||
timeout=timeout,
|
||||
)
|
||||
writer.close()
|
||||
try:
|
||||
await writer.wait_closed()
|
||||
except Exception:
|
||||
pass
|
||||
return True, "tls handshake ok"
|
||||
except asyncio.TimeoutError:
|
||||
return False, "timeout"
|
||||
except (OSError, ssl.SSLError) as exc:
|
||||
return False, f"{type(exc).__name__}: {exc}"
|
||||
|
||||
|
||||
async def _proxy_connect(endpoint: Endpoint, timeout: float) -> Tuple[bool, str]:
|
||||
if endpoint.proxy is None:
|
||||
return await _tcp_probe(endpoint.name, endpoint.port, timeout=timeout)
|
||||
@@ -214,6 +241,59 @@ async def _check_negative(endpoints: List[Endpoint],
|
||||
return leaks
|
||||
|
||||
|
||||
async def _check_host_connectivity(
|
||||
peers: List[Tuple[str, str]], timeout: float
|
||||
) -> None:
|
||||
checks = build_host_checks(peers, HOST_TCP_PORTS, HOST_UDP_PORTS)
|
||||
LOG.info("--- host connectivity checks (%d probes) ---", len(checks))
|
||||
for name, address, protocol, port in checks:
|
||||
if protocol == "tcp":
|
||||
ok, detail = await tcp_probe(address, port, timeout)
|
||||
else:
|
||||
ok, detail = await udp_probe(address, port, timeout)
|
||||
label = f"{name} {address}:{port}/{protocol}"
|
||||
if ok:
|
||||
LOG.info("PASS-HOST %s %s", label, detail)
|
||||
else:
|
||||
LOG.error("FAIL-HOST %s %s", label, detail)
|
||||
|
||||
ok, detail = await _tls_probe(
|
||||
DIRECT_INTERNET_HOST,
|
||||
DIRECT_INTERNET_PORT,
|
||||
max(timeout, HTTP_CONNECT_TIMEOUT_S),
|
||||
)
|
||||
label = f"{DIRECT_INTERNET_HOST}:{DIRECT_INTERNET_PORT}"
|
||||
if ok:
|
||||
LOG.info("PASS-HOST %s direct %s", label, detail)
|
||||
else:
|
||||
LOG.error("FAIL-HOST %s direct %s", label, detail)
|
||||
|
||||
|
||||
def _check_nfs() -> bool:
|
||||
path = Path(NFS_PROBE_PATH)
|
||||
probe = path / (
|
||||
f".connectivity-tester-{socket.gethostname()}-"
|
||||
f"{os.getpid()}-{time.time_ns()}"
|
||||
)
|
||||
payload = f"{socket.gethostname()}:{time.time_ns()}\n"
|
||||
started = time.monotonic()
|
||||
try:
|
||||
probe.write_text(payload, encoding="utf-8")
|
||||
if probe.read_text(encoding="utf-8") != payload:
|
||||
raise OSError("read-back mismatch")
|
||||
LOG.info("PASS-HOST nfs %s read/write latency=%.3fs",
|
||||
NFS_PROBE_PATH, time.monotonic() - started)
|
||||
return True
|
||||
except OSError as exc:
|
||||
LOG.error("FAIL-HOST nfs %s %s", NFS_PROBE_PATH, exc)
|
||||
return False
|
||||
finally:
|
||||
try:
|
||||
probe.unlink()
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
|
||||
async def main() -> int:
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
@@ -222,7 +302,9 @@ async def main() -> int:
|
||||
)
|
||||
period = int(os.environ.get("PERIOD_SECONDS", "600"))
|
||||
timeout = float(os.environ.get("CHECK_TIMEOUT_S", "3"))
|
||||
peers = parse_peers(os.environ.get("SWARM_PEERS", ""))
|
||||
LOG.info("starting; period=%ds timeout=%.1fs", period, timeout)
|
||||
LOG.info("configured swarm peers: %d", len(peers))
|
||||
|
||||
client = docker.DockerClient(base_url="unix://" + _socket_path())
|
||||
self_container = _self_container(client)
|
||||
@@ -264,6 +346,8 @@ async def main() -> int:
|
||||
|
||||
LOG.info("--- negative checks (%d endpoints) ---", len(NEGATIVE))
|
||||
await _check_negative(NEGATIVE, timeout)
|
||||
await _check_host_connectivity(peers, timeout)
|
||||
_check_nfs()
|
||||
except Exception:
|
||||
LOG.exception("round crashed")
|
||||
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import socket
|
||||
from dataclasses import dataclass
|
||||
from typing import Iterable, List, Tuple
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class HostCheck:
|
||||
name: str
|
||||
address: str
|
||||
protocol: str
|
||||
port: int
|
||||
|
||||
|
||||
def parse_peers(value: str) -> List[Tuple[str, str]]:
|
||||
peers = []
|
||||
for item in value.split(","):
|
||||
item = item.strip()
|
||||
if "=" not in item:
|
||||
continue
|
||||
name, address = (part.strip() for part in item.split("=", 1))
|
||||
if name and address:
|
||||
peers.append((name, address))
|
||||
return peers
|
||||
|
||||
|
||||
def build_host_checks(
|
||||
peers: Iterable[Tuple[str, str]],
|
||||
tcp_ports: Iterable[int],
|
||||
udp_ports: Iterable[int],
|
||||
) -> List[Tuple[str, str, str, int]]:
|
||||
checks = []
|
||||
for name, address in peers:
|
||||
checks.extend((name, address, "tcp", port) for port in tcp_ports)
|
||||
checks.extend((name, address, "udp", port) for port in udp_ports)
|
||||
return checks
|
||||
|
||||
|
||||
async def tcp_probe(address: str, port: int, timeout: float) -> Tuple[bool, str]:
|
||||
try:
|
||||
_, writer = await asyncio.wait_for(
|
||||
asyncio.open_connection(address, port), timeout=timeout
|
||||
)
|
||||
writer.close()
|
||||
try:
|
||||
await writer.wait_closed()
|
||||
except Exception:
|
||||
pass
|
||||
return True, "tcp connect ok"
|
||||
except asyncio.TimeoutError:
|
||||
return False, "timeout"
|
||||
except OSError as exc:
|
||||
return False, f"{type(exc).__name__}: {exc}"
|
||||
|
||||
|
||||
async def udp_probe(address: str, port: int, timeout: float) -> Tuple[bool, str]:
|
||||
loop = asyncio.get_running_loop()
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
||||
sock.setblocking(False)
|
||||
try:
|
||||
await asyncio.wait_for(loop.sock_connect(sock, (address, port)), timeout)
|
||||
await asyncio.wait_for(loop.sock_sendall(sock, b"connectivity-tester\n"), timeout)
|
||||
return True, "udp datagram sent; no handshake available"
|
||||
except asyncio.TimeoutError:
|
||||
return False, "timeout"
|
||||
except OSError as exc:
|
||||
return False, f"{type(exc).__name__}: {exc}"
|
||||
finally:
|
||||
sock.close()
|
||||
@@ -73,9 +73,7 @@ NETWORK_MAP: Dict[str, List[Endpoint]] = {
|
||||
"docker_socket_proxy_hermes_proxy": [
|
||||
Endpoint("hermes-gateway", 8000),
|
||||
],
|
||||
"connectivity_tester_default": [
|
||||
Endpoint("connectivity-tester", 8765),
|
||||
],
|
||||
"connectivity_tester_default": [],
|
||||
"brencher_brencher-net": [
|
||||
Endpoint("brencher-backend", 5000),
|
||||
],
|
||||
|
||||
@@ -5,6 +5,7 @@ services:
|
||||
user: "0:0"
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
- hermes_nfs_probe:/mnt/hermes-nfs-probe
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "test -S /var/run/docker.sock"]
|
||||
interval: 60s
|
||||
@@ -23,3 +24,11 @@ services:
|
||||
condition: on-failure
|
||||
delay: 10s
|
||||
max_attempts: 3
|
||||
|
||||
volumes:
|
||||
hermes_nfs_probe:
|
||||
driver: local
|
||||
driver_opts:
|
||||
type: nfs
|
||||
o: addr=192.168.1.96,rw,nfsvers=4
|
||||
device: :/root/volumes/hermes/agent
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
import unittest
|
||||
|
||||
from host_checks import build_host_checks, parse_peers
|
||||
|
||||
|
||||
class HostChecksTests(unittest.TestCase):
|
||||
def test_parse_peers(self):
|
||||
self.assertEqual(
|
||||
parse_peers("odroid=192.168.1.96,rudolf-T=100.105.201.15"),
|
||||
[("odroid", "192.168.1.96"), ("rudolf-T", "100.105.201.15")],
|
||||
)
|
||||
|
||||
def test_parse_peers_ignores_invalid_entries(self):
|
||||
self.assertEqual(
|
||||
parse_peers("odroid=192.168.1.96,broken,rudolf-T="),
|
||||
[("odroid", "192.168.1.96")],
|
||||
)
|
||||
|
||||
def test_build_host_checks(self):
|
||||
checks = build_host_checks(
|
||||
[("odroid", "192.168.1.96"), ("rudolf-T", "100.105.201.15")],
|
||||
[2377, 7946],
|
||||
[7946, 4789],
|
||||
)
|
||||
self.assertEqual(
|
||||
checks,
|
||||
[
|
||||
("odroid", "192.168.1.96", "tcp", 2377),
|
||||
("odroid", "192.168.1.96", "tcp", 7946),
|
||||
("odroid", "192.168.1.96", "udp", 7946),
|
||||
("odroid", "192.168.1.96", "udp", 4789),
|
||||
("rudolf-T", "100.105.201.15", "tcp", 2377),
|
||||
("rudolf-T", "100.105.201.15", "tcp", 7946),
|
||||
("rudolf-T", "100.105.201.15", "udp", 7946),
|
||||
("rudolf-T", "100.105.201.15", "udp", 4789),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user