feat(connectivity): probe swarm host ports and Telegram egress
This commit is contained in:
@@ -23,11 +23,14 @@ from docker.models.containers import Container
|
||||
from docker.models.networks import Network
|
||||
|
||||
from network_map import Endpoint, GLOBAL_SERVICES, NEGATIVE, NETWORK_MAP, PUBLIC_SERVICES
|
||||
from host_checks import build_host_checks, parse_peers, tcp_probe, udp_probe
|
||||
|
||||
LOG = logging.getLogger("connectivity-tester")
|
||||
|
||||
FALLBACK_TCP_TIMEOUT_S = 3.0
|
||||
HTTP_CONNECT_TIMEOUT_S = 5.0
|
||||
HOST_TCP_PORTS = (2377, 7946, 2049)
|
||||
HOST_UDP_PORTS = (7946, 4789)
|
||||
|
||||
|
||||
def _socket_path() -> str:
|
||||
@@ -214,6 +217,27 @@ async def _check_negative(endpoints: List[Endpoint],
|
||||
return leaks
|
||||
|
||||
|
||||
async def _check_host_connectivity(
|
||||
peers: List[Tuple[str, str]], timeout: float
|
||||
) -> None:
|
||||
checks = build_host_checks(peers, HOST_TCP_PORTS, HOST_UDP_PORTS)
|
||||
LOG.info("--- host connectivity checks (%d probes) ---", len(checks))
|
||||
for name, address, protocol, port in checks:
|
||||
if protocol == "tcp":
|
||||
ok, detail = await tcp_probe(address, port, timeout)
|
||||
else:
|
||||
ok, detail = await udp_probe(address, port, timeout)
|
||||
label = f"{name} {address}:{port}/{protocol}"
|
||||
if ok:
|
||||
LOG.info("PASS-HOST %s %s", label, detail)
|
||||
else:
|
||||
LOG.error("FAIL-HOST %s %s", label, detail)
|
||||
await _check_endpoint(
|
||||
Endpoint("api.telegram.org", 443),
|
||||
max(timeout, HTTP_CONNECT_TIMEOUT_S),
|
||||
)
|
||||
|
||||
|
||||
async def main() -> int:
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
@@ -222,7 +246,9 @@ async def main() -> int:
|
||||
)
|
||||
period = int(os.environ.get("PERIOD_SECONDS", "600"))
|
||||
timeout = float(os.environ.get("CHECK_TIMEOUT_S", "3"))
|
||||
peers = parse_peers(os.environ.get("SWARM_PEERS", ""))
|
||||
LOG.info("starting; period=%ds timeout=%.1fs", period, timeout)
|
||||
LOG.info("configured swarm peers: %d", len(peers))
|
||||
|
||||
client = docker.DockerClient(base_url="unix://" + _socket_path())
|
||||
self_container = _self_container(client)
|
||||
@@ -264,6 +290,7 @@ async def main() -> int:
|
||||
|
||||
LOG.info("--- negative checks (%d endpoints) ---", len(NEGATIVE))
|
||||
await _check_negative(NEGATIVE, timeout)
|
||||
await _check_host_connectivity(peers, timeout)
|
||||
except Exception:
|
||||
LOG.exception("round crashed")
|
||||
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import socket
|
||||
from dataclasses import dataclass
|
||||
from typing import Iterable, List, Tuple
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class HostCheck:
|
||||
name: str
|
||||
address: str
|
||||
protocol: str
|
||||
port: int
|
||||
|
||||
|
||||
def parse_peers(value: str) -> List[Tuple[str, str]]:
|
||||
peers = []
|
||||
for item in value.split(","):
|
||||
item = item.strip()
|
||||
if "=" not in item:
|
||||
continue
|
||||
name, address = (part.strip() for part in item.split("=", 1))
|
||||
if name and address:
|
||||
peers.append((name, address))
|
||||
return peers
|
||||
|
||||
|
||||
def build_host_checks(
|
||||
peers: Iterable[Tuple[str, str]],
|
||||
tcp_ports: Iterable[int],
|
||||
udp_ports: Iterable[int],
|
||||
) -> List[Tuple[str, str, str, int]]:
|
||||
checks = []
|
||||
for name, address in peers:
|
||||
checks.extend((name, address, "tcp", port) for port in tcp_ports)
|
||||
checks.extend((name, address, "udp", port) for port in udp_ports)
|
||||
return checks
|
||||
|
||||
|
||||
async def tcp_probe(address: str, port: int, timeout: float) -> Tuple[bool, str]:
|
||||
try:
|
||||
_, writer = await asyncio.wait_for(
|
||||
asyncio.open_connection(address, port), timeout=timeout
|
||||
)
|
||||
writer.close()
|
||||
try:
|
||||
await writer.wait_closed()
|
||||
except Exception:
|
||||
pass
|
||||
return True, "tcp connect ok"
|
||||
except asyncio.TimeoutError:
|
||||
return False, "timeout"
|
||||
except OSError as exc:
|
||||
return False, f"{type(exc).__name__}: {exc}"
|
||||
|
||||
|
||||
async def udp_probe(address: str, port: int, timeout: float) -> Tuple[bool, str]:
|
||||
loop = asyncio.get_running_loop()
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
||||
sock.setblocking(False)
|
||||
try:
|
||||
await asyncio.wait_for(loop.sock_connect(sock, (address, port)), timeout)
|
||||
await asyncio.wait_for(loop.sock_sendall(sock, b"connectivity-tester\n"), timeout)
|
||||
return True, "udp datagram sent; no handshake available"
|
||||
except asyncio.TimeoutError:
|
||||
return False, "timeout"
|
||||
except OSError as exc:
|
||||
return False, f"{type(exc).__name__}: {exc}"
|
||||
finally:
|
||||
sock.close()
|
||||
@@ -3,6 +3,8 @@ services:
|
||||
image: gitea.rudolf.keenetic.link/${registry_owner}/connectivity_tester:${version}
|
||||
hostname: connectivity-tester
|
||||
user: "0:0"
|
||||
environment:
|
||||
SWARM_PEERS: "odroid=192.168.1.96,rudolf-T=100.105.201.15,xu4=192.168.1.114"
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
healthcheck:
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
import unittest
|
||||
|
||||
from host_checks import build_host_checks, parse_peers
|
||||
|
||||
|
||||
class HostChecksTests(unittest.TestCase):
|
||||
def test_parse_peers(self):
|
||||
self.assertEqual(
|
||||
parse_peers("odroid=192.168.1.96,rudolf-T=100.105.201.15"),
|
||||
[("odroid", "192.168.1.96"), ("rudolf-T", "100.105.201.15")],
|
||||
)
|
||||
|
||||
def test_parse_peers_ignores_invalid_entries(self):
|
||||
self.assertEqual(
|
||||
parse_peers("odroid=192.168.1.96,broken,rudolf-T="),
|
||||
[("odroid", "192.168.1.96")],
|
||||
)
|
||||
|
||||
def test_build_host_checks(self):
|
||||
checks = build_host_checks(
|
||||
[("odroid", "192.168.1.96"), ("rudolf-T", "100.105.201.15")],
|
||||
[2377, 7946],
|
||||
[7946, 4789],
|
||||
)
|
||||
self.assertEqual(
|
||||
checks,
|
||||
[
|
||||
("odroid", "192.168.1.96", "tcp", 2377),
|
||||
("odroid", "192.168.1.96", "tcp", 7946),
|
||||
("odroid", "192.168.1.96", "udp", 7946),
|
||||
("odroid", "192.168.1.96", "udp", 4789),
|
||||
("rudolf-T", "100.105.201.15", "tcp", 2377),
|
||||
("rudolf-T", "100.105.201.15", "tcp", 7946),
|
||||
("rudolf-T", "100.105.201.15", "udp", 7946),
|
||||
("rudolf-T", "100.105.201.15", "udp", 4789),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user