Compare commits
36
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e6444ad49c | ||
|
|
eb851d9b1d | ||
|
|
c719f6ab8d | ||
|
|
6e6f9c67fc | ||
|
|
77b3cc2c08 | ||
|
|
d011648e56 | ||
|
|
c22cadc48b | ||
|
|
2fa05da943 | ||
|
|
3406491dcc | ||
|
|
7a08d4d022 | ||
|
|
6e9a998eba | ||
|
|
ef4af24c1e | ||
|
|
f80a524d33 | ||
|
|
faa3e32a53 | ||
|
|
e6d184bd95 | ||
|
|
d097862c81 | ||
|
|
b2e85b686c | ||
|
|
932be82ae6 | ||
|
|
953e4dcba7 | ||
|
|
81e0641ab1 | ||
|
|
8f71c723fb | ||
|
|
7fed0c08cd | ||
|
|
4b8ece95c0 | ||
|
|
0734adc229 | ||
|
|
8ecc6f0424 | ||
|
|
4ab39982d1 | ||
|
|
b5b94b2eee | ||
|
|
bf6b86302f | ||
|
|
1e9d33485d | ||
|
|
d5b1a23bcb | ||
|
|
e94ea37e85 | ||
|
|
c0d53b709c | ||
|
|
0366a78a96 | ||
|
|
c13e9d2ce7 | ||
|
|
020a5092a0 | ||
|
|
5a36921f3b |
@@ -0,0 +1,49 @@
|
||||
name: release
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- 'auto-*'
|
||||
tags:
|
||||
- 'v*'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
tag:
|
||||
description: 'Image tag (e.g. auto-<sha>)'
|
||||
required: false
|
||||
default: latest
|
||||
type: string
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: log into Gitea Packages
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: gitea.rudolf.keenetic.link
|
||||
username: ${{ secrets.REGISTRY_USERNAME }}
|
||||
password: ${{ secrets.REGISTRY_TOKEN }}
|
||||
|
||||
- name: resolve version
|
||||
id: version
|
||||
run: |
|
||||
if [ -n "${{ inputs.tag }}" ] && [ "${{ inputs.tag }}" != "latest" ]; then
|
||||
echo "VERSION=${{ inputs.tag }}" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "VERSION=${GITHUB_REF_NAME##*/}" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
echo "VERSION=$VERSION"
|
||||
|
||||
- name: build and push
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: connectivity_tester
|
||||
file: connectivity_tester/Dockerfile
|
||||
push: true
|
||||
tags: |
|
||||
gitea.rudolf.keenetic.link/hermes/connectivity_tester:${{ steps.version.outputs.VERSION }}
|
||||
gitea.rudolf.keenetic.link/hermes/connectivity_tester:latest
|
||||
@@ -0,0 +1,12 @@
|
||||
FROM python:3.12-slim
|
||||
|
||||
WORKDIR /app
|
||||
COPY . /app/
|
||||
|
||||
RUN pip install --no-cache-dir docker==7.1.0
|
||||
|
||||
ENV PYTHONUNBUFFERED=1 \
|
||||
PERIOD_SECONDS=600 \
|
||||
CHECK_TIMEOUT_S=3
|
||||
|
||||
CMD ["python", "-m", "app"]
|
||||
@@ -0,0 +1,276 @@
|
||||
"""Connectivity tester.
|
||||
|
||||
For every configured overlay network the container attaches itself, runs
|
||||
TCP checks against its endpoints, then detaches. Endpoints marked with
|
||||
``proxy=(host, port)`` are probed through that HTTP proxy (CONNECT). After
|
||||
each round a final loop checks PUBLIC_SERVICES. Cross-validation:
|
||||
- service reachable but not in map -> ERROR 'LEAK-UNEXPECTED'
|
||||
- service in map but unreachable -> ERROR 'MISSING-EXPECTED'
|
||||
- global service not in any map -> WARNING 'GLOBAL-UNCOVERED'
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import socket
|
||||
import sys
|
||||
import time
|
||||
from typing import Iterable, List, Set, Tuple
|
||||
|
||||
import docker
|
||||
from docker.models.containers import Container
|
||||
from docker.models.networks import Network
|
||||
|
||||
from network_map import Endpoint, GLOBAL_SERVICES, NEGATIVE, NETWORK_MAP, PUBLIC_SERVICES
|
||||
|
||||
LOG = logging.getLogger("connectivity-tester")
|
||||
|
||||
FALLBACK_TCP_TIMEOUT_S = 3.0
|
||||
HTTP_CONNECT_TIMEOUT_S = 5.0
|
||||
|
||||
|
||||
def _socket_path() -> str:
|
||||
return os.environ.get("DOCKER_SOCKET", "/var/run/docker.sock")
|
||||
|
||||
|
||||
def _self_container(client: docker.DockerClient) -> Container:
|
||||
hostname = socket.gethostname()
|
||||
for c in client.containers.list():
|
||||
cn = c.name.lstrip("/")
|
||||
hn = c.attrs.get("Config", {}).get("Hostname", "")
|
||||
if hostname == cn or hostname == hn or hostname[:12] == c.id[:12]:
|
||||
return c
|
||||
candidates = [c for c in client.containers.list() if hostname[:12] == c.id]
|
||||
if len(candidates) == 1:
|
||||
return candidates[0]
|
||||
raise RuntimeError(f"self container not found for hostname={hostname}")
|
||||
|
||||
|
||||
def _swarm_networks_for(client: docker.DockerClient,
|
||||
names: Iterable[str]) -> dict:
|
||||
out = {}
|
||||
by_name = {n.name: n for n in client.networks.list()}
|
||||
for name in names:
|
||||
net = by_name.get(name)
|
||||
if net is not None and net.attrs.get("Scope") == "swarm":
|
||||
out[name] = net
|
||||
return out
|
||||
|
||||
|
||||
def _attach(client: docker.DockerClient, container: Container,
|
||||
network: Network) -> bool:
|
||||
current = (container.attrs.get("NetworkSettings", {}) or {}).get("Networks", {})
|
||||
if network.name in current:
|
||||
return False
|
||||
try:
|
||||
network.connect(container, aliases=["connectivity-tester",
|
||||
socket.gethostname()])
|
||||
return True
|
||||
except Exception as exc:
|
||||
if "already exists" in str(exc).lower():
|
||||
return False
|
||||
raise
|
||||
|
||||
|
||||
def _detach(client: docker.DockerClient, container: Container,
|
||||
network: Network) -> None:
|
||||
try:
|
||||
network.disconnect(container, force=True)
|
||||
except Exception as exc:
|
||||
if "not found" in str(exc).lower():
|
||||
return
|
||||
raise
|
||||
|
||||
|
||||
async def _tcp_probe(host: str, port: int, timeout: float) -> Tuple[bool, str]:
|
||||
try:
|
||||
fut = asyncio.open_connection(host=host, port=port)
|
||||
reader, writer = await asyncio.wait_for(fut, timeout=timeout)
|
||||
writer.close()
|
||||
try:
|
||||
await writer.wait_closed()
|
||||
except Exception:
|
||||
pass
|
||||
return True, "tcp ok"
|
||||
except asyncio.TimeoutError:
|
||||
return False, "timeout"
|
||||
except OSError as exc:
|
||||
return False, f"{type(exc).__name__}: {exc}"
|
||||
|
||||
|
||||
async def _proxy_connect(endpoint: Endpoint, timeout: float) -> Tuple[bool, str]:
|
||||
if endpoint.proxy is None:
|
||||
return await _tcp_probe(endpoint.name, endpoint.port, timeout=timeout)
|
||||
proxy_host, proxy_port = endpoint.proxy
|
||||
target = endpoint.name
|
||||
target_port = endpoint.port
|
||||
try:
|
||||
fut = asyncio.open_connection(host=proxy_host, port=proxy_port)
|
||||
reader, writer = await asyncio.wait_for(fut, timeout=timeout)
|
||||
req = (
|
||||
f"CONNECT {target}:{target_port} HTTP/1.1\r\n"
|
||||
f"Host: {target}:{target_port}\r\n\r\n"
|
||||
)
|
||||
writer.write(req.encode("ascii"))
|
||||
await writer.drain()
|
||||
status_line = await asyncio.wait_for(reader.readline(),
|
||||
timeout=timeout)
|
||||
if not status_line.startswith(b"HTTP/") or b" 200 " not in status_line:
|
||||
writer.close()
|
||||
try:
|
||||
await writer.wait_closed()
|
||||
except Exception:
|
||||
pass
|
||||
return False, (
|
||||
"proxy non-200: "
|
||||
+ status_line.strip().decode(errors="replace")
|
||||
)
|
||||
while True:
|
||||
line = await asyncio.wait_for(reader.readline(), timeout=timeout)
|
||||
if line in (b"\r\n", b"", b"\n"):
|
||||
break
|
||||
writer.close()
|
||||
try:
|
||||
await writer.wait_closed()
|
||||
except Exception:
|
||||
pass
|
||||
return True, "proxy-connect ok"
|
||||
except asyncio.TimeoutError:
|
||||
return False, "timeout"
|
||||
except OSError as exc:
|
||||
return False, f"{type(exc).__name__}: {exc}"
|
||||
|
||||
|
||||
async def _check_endpoint(endpoint: Endpoint, timeout: float) -> bool:
|
||||
if endpoint.proxy is not None:
|
||||
ok, detail = await _proxy_connect(endpoint, timeout)
|
||||
else:
|
||||
ok, detail = await _tcp_probe(endpoint.name, endpoint.port,
|
||||
timeout=timeout)
|
||||
label = f"{endpoint.name}:{endpoint.port}"
|
||||
src = f"via {endpoint.proxy}" if endpoint.proxy else "direct"
|
||||
if ok:
|
||||
LOG.info("PASS %s %s %s", label, src, detail)
|
||||
else:
|
||||
LOG.error("FAIL %s %s %s", label, src, detail)
|
||||
return ok
|
||||
|
||||
|
||||
async def _check_network(network_name: str,
|
||||
endpoints: List[Endpoint],
|
||||
timeout: float) -> Tuple[Set[Endpoint], Set[Endpoint]]:
|
||||
reachable: Set[Endpoint] = set()
|
||||
unreachable: Set[Endpoint] = set()
|
||||
for ep in endpoints:
|
||||
ok = await _check_endpoint(ep, timeout)
|
||||
(reachable if ok else unreachable).add(ep)
|
||||
return reachable, unreachable
|
||||
|
||||
|
||||
def _cross_validate(network_name: str,
|
||||
mapped: List[Endpoint],
|
||||
reachable: Set[Endpoint],
|
||||
unreachable: Set[Endpoint],
|
||||
global_set: Set[Endpoint]) -> None:
|
||||
mapped_set = set(mapped)
|
||||
reachable_global = reachable & global_set
|
||||
for ep in reachable_global - mapped_set:
|
||||
LOG.error("LEAK-UNEXPECTED %s reachable in %s but not in map",
|
||||
f"{ep.name}:{ep.port}", network_name)
|
||||
for ep in mapped_set & unreachable:
|
||||
LOG.error("MISSING-EXPECTED %s unreachable in %s but listed in map",
|
||||
f"{ep.name}:{ep.port}", network_name)
|
||||
for ep in mapped_set & reachable:
|
||||
LOG.info("OK-MAPPED %s ok in %s", f"{ep.name}:{ep.port}", network_name)
|
||||
for ep in mapped_set - reachable - unreachable:
|
||||
LOG.warning("NOT-CHECKED %s in map for %s",
|
||||
f"{ep.name}:{ep.port}", network_name)
|
||||
|
||||
|
||||
def _consistency_audit(net_names: List[str]) -> None:
|
||||
flat: Set[Endpoint] = set()
|
||||
for n in net_names:
|
||||
flat.update(NETWORK_MAP.get(n, []))
|
||||
covered = flat | set(PUBLIC_SERVICES)
|
||||
all_global = set(GLOBAL_SERVICES)
|
||||
uncovered = all_global - covered
|
||||
if uncovered:
|
||||
LOG.warning("GLOBAL-UNCOVERED %s not present in any map",
|
||||
sorted(f"{e.name}:{e.port}" for e in uncovered))
|
||||
|
||||
|
||||
async def _check_negative(endpoints: List[Endpoint],
|
||||
timeout: float) -> int:
|
||||
leaks = 0
|
||||
for ep in endpoints:
|
||||
ok = await _tcp_probe(ep.name, ep.port, timeout=timeout)
|
||||
if ok:
|
||||
LOG.error("LEAK %s:%d reachable but must NOT be",
|
||||
ep.name, ep.port)
|
||||
leaks += 1
|
||||
else:
|
||||
LOG.info("OK-NEG %s:%d blocked", ep.name, ep.port)
|
||||
return leaks
|
||||
|
||||
|
||||
async def main() -> int:
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s %(levelname)s %(message)s",
|
||||
stream=sys.stdout,
|
||||
)
|
||||
period = int(os.environ.get("PERIOD_SECONDS", "600"))
|
||||
timeout = float(os.environ.get("CHECK_TIMEOUT_S", "3"))
|
||||
LOG.info("starting; period=%ds timeout=%.1fs", period, timeout)
|
||||
|
||||
client = docker.DockerClient(base_url="unix://" + _socket_path())
|
||||
self_container = _self_container(client)
|
||||
|
||||
nets = _swarm_networks_for(client, NETWORK_MAP.keys())
|
||||
planned = list(NETWORK_MAP.keys())
|
||||
LOG.info("configured networks: %d, present in swarm: %d",
|
||||
len(planned), len(nets))
|
||||
for n in nets:
|
||||
LOG.info("plan network: %s", n)
|
||||
|
||||
_consistency_audit(planned)
|
||||
|
||||
global_set: Set[Endpoint] = set(GLOBAL_SERVICES)
|
||||
|
||||
while True:
|
||||
started = time.monotonic()
|
||||
try:
|
||||
for net_name, net in nets.items():
|
||||
eps_in_map: List[Endpoint] = [
|
||||
e for e in NETWORK_MAP[net_name] if e.proxy is None
|
||||
]
|
||||
LOG.info("--- round begin, attaching to %s (%d endpoints) ---",
|
||||
net_name, len(eps_in_map))
|
||||
_attach(client, self_container, net)
|
||||
try:
|
||||
reachable, unreachable = await _check_network(
|
||||
net_name, eps_in_map, timeout)
|
||||
_cross_validate(net_name, eps_in_map,
|
||||
reachable, unreachable, global_set)
|
||||
finally:
|
||||
_detach(client, self_container, net)
|
||||
LOG.info("--- round end on %s ---", net_name)
|
||||
|
||||
LOG.info("--- public checks (%d endpoints) ---",
|
||||
len(PUBLIC_SERVICES))
|
||||
for ep in PUBLIC_SERVICES:
|
||||
await _check_endpoint(ep, max(timeout, HTTP_CONNECT_TIMEOUT_S))
|
||||
|
||||
LOG.info("--- negative checks (%d endpoints) ---", len(NEGATIVE))
|
||||
await _check_negative(NEGATIVE, timeout)
|
||||
except Exception:
|
||||
LOG.exception("round crashed")
|
||||
|
||||
elapsed = time.monotonic() - started
|
||||
sleep_for = max(0.0, period - elapsed)
|
||||
await asyncio.sleep(sleep_for)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,122 @@
|
||||
from dataclasses import dataclass
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Endpoint:
|
||||
name: str
|
||||
port: int
|
||||
proxy: Optional[Tuple[str, int]] = None
|
||||
tls: bool = False
|
||||
|
||||
|
||||
SWARM_NETWORKS = (
|
||||
"hermes_infra_hermes_proxy",
|
||||
"hermes_infra_squid_egress",
|
||||
"hermes_infra_admin",
|
||||
"hermes_infra_default",
|
||||
"hermes_hermes",
|
||||
"gitea_gitea",
|
||||
"gmail_mcp_default",
|
||||
"immich_default",
|
||||
"torrserv_proxy_default",
|
||||
"docker_socket_proxy_hermes_proxy",
|
||||
"connectivity_tester_default",
|
||||
"brencher_brencher-net",
|
||||
"brencher2_brencher-net",
|
||||
)
|
||||
|
||||
|
||||
SQUID_PROXY: Tuple[str, int] = ("squid", 3128)
|
||||
|
||||
|
||||
NETWORK_MAP: Dict[str, List[Endpoint]] = {
|
||||
"hermes_infra_hermes_proxy": [
|
||||
Endpoint("hermes-gateway", 8000),
|
||||
Endpoint("hermes-ui", 8087),
|
||||
Endpoint("squid", 3128),
|
||||
Endpoint("docker_socket_proxy", 2375),
|
||||
Endpoint("brencher-state-proxy", 80),
|
||||
],
|
||||
"hermes_infra_squid_egress": [
|
||||
Endpoint("squid", 3128),
|
||||
],
|
||||
"hermes_infra_admin": [
|
||||
Endpoint("sarg", 80),
|
||||
],
|
||||
"hermes_infra_default": [
|
||||
Endpoint("hermes-ui", 8087),
|
||||
],
|
||||
"hermes_hermes": [
|
||||
Endpoint("hermes-agent", 8000),
|
||||
Endpoint("hermes-gateway", 8000),
|
||||
],
|
||||
"gitea_gitea": [
|
||||
Endpoint("gitea-server", 3000),
|
||||
Endpoint("act_runner", 8088),
|
||||
Endpoint("cache_server", 37051),
|
||||
],
|
||||
"gmail_mcp_default": [
|
||||
Endpoint("gmail-mcp-server", 8000),
|
||||
Endpoint("telegram-mcp-server", 8000),
|
||||
],
|
||||
"immich_default": [
|
||||
Endpoint("immich-server", 2283),
|
||||
Endpoint("immich-database", 5432),
|
||||
Endpoint("immich-redis", 6379),
|
||||
Endpoint("immich-machine-learning", 8501),
|
||||
],
|
||||
"torrserv_proxy_default": [
|
||||
Endpoint("fileserver", 8000),
|
||||
Endpoint("reverseproxy", 8080),
|
||||
],
|
||||
"docker_socket_proxy_hermes_proxy": [
|
||||
Endpoint("hermes-gateway", 8000),
|
||||
],
|
||||
"connectivity_tester_default": [
|
||||
Endpoint("connectivity-tester", 8765),
|
||||
],
|
||||
"brencher_brencher-net": [
|
||||
Endpoint("brencher-backend", 5000),
|
||||
],
|
||||
"brencher2_brencher-net": [
|
||||
Endpoint("brencher-backend", 5000),
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
PUBLIC_SERVICES: List[Endpoint] = [
|
||||
Endpoint("gitea.rudolf.keenetic.link", 443, proxy=SQUID_PROXY, tls=True),
|
||||
Endpoint("gitea.rudolf.keenetic.link", 80, proxy=SQUID_PROXY),
|
||||
Endpoint("registry.rudolf.keenetic.link", 443, proxy=SQUID_PROXY, tls=True),
|
||||
Endpoint("registry.rudolf.keenetic.link", 5000, proxy=SQUID_PROXY),
|
||||
Endpoint("deb.debian.org", 80, proxy=SQUID_PROXY),
|
||||
Endpoint("deb.debian.org", 443, proxy=SQUID_PROXY, tls=True),
|
||||
Endpoint("github.com", 443, proxy=SQUID_PROXY, tls=True),
|
||||
Endpoint("githubusercontent.com", 443, proxy=SQUID_PROXY, tls=True),
|
||||
Endpoint("telegram.org", 443, proxy=SQUID_PROXY, tls=True),
|
||||
Endpoint("t.me", 443, proxy=SQUID_PROXY, tls=True),
|
||||
Endpoint("minimax.io", 443, proxy=SQUID_PROXY, tls=True),
|
||||
Endpoint("minimax.chat", 443, proxy=SQUID_PROXY, tls=True),
|
||||
Endpoint("minimax.com", 443, proxy=SQUID_PROXY, tls=True),
|
||||
Endpoint("pypi.org", 443, proxy=SQUID_PROXY, tls=True),
|
||||
Endpoint("google.com", 443, proxy=SQUID_PROXY, tls=True),
|
||||
Endpoint("cloudflare.com", 443, proxy=SQUID_PROXY, tls=True),
|
||||
Endpoint("wikipedia.org", 443, proxy=SQUID_PROXY, tls=True),
|
||||
Endpoint("mozilla.org", 443, proxy=SQUID_PROXY, tls=True),
|
||||
Endpoint("1.1.1.1", 443, proxy=SQUID_PROXY, tls=True),
|
||||
Endpoint("8.8.8.8", 53),
|
||||
Endpoint("8.8.4.4", 53),
|
||||
]
|
||||
|
||||
|
||||
GLOBAL_SERVICES: List[Endpoint] = sorted(
|
||||
{ep for eps in NETWORK_MAP.values() for ep in eps} | set(PUBLIC_SERVICES),
|
||||
key=lambda e: (e.name, e.port),
|
||||
)
|
||||
|
||||
|
||||
NEGATIVE: List[Endpoint] = [
|
||||
Endpoint("169.254.169.254", 80),
|
||||
Endpoint("metadata.google.internal", 80),
|
||||
]
|
||||
@@ -0,0 +1,25 @@
|
||||
services:
|
||||
connectivity-tester:
|
||||
image: gitea.rudolf.keenetic.link/hermes/connectivity_tester:${version}
|
||||
hostname: connectivity-tester
|
||||
user: "0:0"
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "test -S /var/run/docker.sock"]
|
||||
interval: 60s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 30s
|
||||
deploy:
|
||||
mode: global
|
||||
resources:
|
||||
limits:
|
||||
memory: 128M
|
||||
pids: 64
|
||||
reservations:
|
||||
memory: 64M
|
||||
restart_policy:
|
||||
condition: on-failure
|
||||
delay: 10s
|
||||
max_attempts: 3
|
||||
Reference in New Issue
Block a user