This commit is contained in:
+1
-1
@@ -1 +1 @@
|
||||
__pycache__/
|
||||
**/__pycache__/
|
||||
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,20 @@
|
||||
# Deploy
|
||||
|
||||
```sh
|
||||
kubectl apply -k k3s
|
||||
kubectl -n connectivity-test rollout status statefulset/k3s-connectivity
|
||||
```
|
||||
|
||||
# Check
|
||||
|
||||
```sh
|
||||
kubectl -n connectivity-test get pods -o wide
|
||||
kubectl -n connectivity-test exec k3s-connectivity-0 -- \
|
||||
python -c "from urllib.request import urlopen; print(urlopen('http://localhost:8080/check').read().decode())"
|
||||
```
|
||||
|
||||
# Undeploy
|
||||
|
||||
```sh
|
||||
kubectl delete -k k3s
|
||||
```
|
||||
+20
-105
@@ -1,108 +1,23 @@
|
||||
# K3s Cross-Node Connectivity Tester
|
||||
|
||||
This workload checks HTTP connectivity between StatefulSet pods by their
|
||||
individual internal DNS names. A successful `/probe` response includes the
|
||||
source and peer node names and reports `all_cross_node: true` only when every
|
||||
peer responded from a different node.
|
||||
## Functional Requirements
|
||||
|
||||
The default deployment uses two replicas and HTTP on port `8080`.
|
||||
|
||||
## Build and distribute the image
|
||||
|
||||
The included StatefulSet pulls:
|
||||
|
||||
```text
|
||||
gitea.rudolf.keenetic.link/org1/swarm_connectivity_tester:latest
|
||||
```
|
||||
|
||||
To build a replacement image locally:
|
||||
|
||||
```sh
|
||||
docker build -t k3s-connectivity-tester:0.1.0 .
|
||||
```
|
||||
|
||||
The Gitea workflow publishes both `linux/amd64` and `linux/arm64` variants. In
|
||||
the Gitea repository's Actions secrets, configure:
|
||||
|
||||
- `REGISTRY_USERNAME`: a Gitea user or bot account with package write access.
|
||||
- `REGISTRY_TOKEN`: that account's access token with package write access.
|
||||
|
||||
Push the workflow update, wait for the `Build K3s Connectivity Image` action to
|
||||
finish, then restart the StatefulSet to pull the multi-architecture image:
|
||||
|
||||
```sh
|
||||
kubectl -n connectivity-test rollout restart statefulset/k3s-connectivity
|
||||
kubectl -n connectivity-test rollout status statefulset/k3s-connectivity
|
||||
```
|
||||
|
||||
For a multi-node K3s cluster, the image registry must be reachable from every
|
||||
node. Alternatively, import a local image into the K3s containerd image store
|
||||
on every node:
|
||||
|
||||
```sh
|
||||
docker save k3s-connectivity-tester:0.1.0 | sudo k3s ctr images import -
|
||||
```
|
||||
|
||||
## Deploy and verify HTTP
|
||||
|
||||
```sh
|
||||
kubectl apply -k .
|
||||
kubectl -n connectivity-test rollout status statefulset/k3s-connectivity
|
||||
kubectl -n connectivity-test get pods -o wide
|
||||
```
|
||||
|
||||
The two pods must show different values in the `NODE` column. This is enforced
|
||||
by required pod anti-affinity and a `DoNotSchedule` hostname spread rule. On a
|
||||
single-node cluster, the second pod deliberately remains Pending rather than
|
||||
silently testing same-node traffic.
|
||||
|
||||
Nodes carrying the `node.kubernetes.io/disk-pressure:NoSchedule` taint are not
|
||||
used. Resolve their disk pressure before expecting the second pod to schedule;
|
||||
do not remove that taint just to run this diagnostic workload.
|
||||
|
||||
Run an immediate probe from the first pod:
|
||||
|
||||
```sh
|
||||
kubectl -n connectivity-test exec k3s-connectivity-0 -- \
|
||||
python -c "from urllib.request import urlopen; print(urlopen('http://localhost:8080/probe').read().decode())"
|
||||
```
|
||||
|
||||
The response contains a peer URL such as:
|
||||
|
||||
```text
|
||||
http://k3s-connectivity-1.k3s-connectivity.connectivity-test.svc.cluster.local:8080/healthz
|
||||
```
|
||||
|
||||
Check the latest periodic result without triggering another request:
|
||||
|
||||
```sh
|
||||
kubectl -n connectivity-test exec k3s-connectivity-0 -- \
|
||||
python -c "from urllib.request import urlopen; print(urlopen('http://localhost:8080/results').read().decode())"
|
||||
```
|
||||
|
||||
`all_reachable` and `all_cross_node` must both be `true`. The response also
|
||||
includes the exact source and peer node names, which makes failed placement or
|
||||
network-policy diagnosis straightforward.
|
||||
|
||||
## Enable HTTPS
|
||||
|
||||
Create a TLS Secret in the workload namespace, then apply the included
|
||||
strategic merge patch:
|
||||
|
||||
```sh
|
||||
kubectl -n connectivity-test create secret tls k3s-connectivity-tls \
|
||||
--cert=tls.crt --key=tls.key
|
||||
kubectl -n connectivity-test patch statefulset k3s-connectivity \
|
||||
--type=strategic --patch-file=https-patch.yaml
|
||||
kubectl -n connectivity-test rollout status statefulset/k3s-connectivity
|
||||
```
|
||||
|
||||
The HTTPS patch disables certificate verification for a self-signed diagnostic
|
||||
certificate. For a certificate issued by a trusted internal CA with SANs for
|
||||
the pod DNS names, set `VERIFY_TLS` to `true` in the StatefulSet.
|
||||
|
||||
To remove the test:
|
||||
|
||||
```sh
|
||||
kubectl delete -k .
|
||||
```
|
||||
- Run one tester pod on each of the three target nodes, with required hostname
|
||||
anti-affinity preventing more than one tester pod on any node.
|
||||
- Use parallel StatefulSet replicas with stable internal names:
|
||||
`k3s-connectivity-0`, `k3s-connectivity-1`, and `k3s-connectivity-2`.
|
||||
- Probe every other replica over HTTP by its internal Kubernetes DNS name, such
|
||||
as `k3s-connectivity-1.k3s-connectivity.connectivity-test.svc.cluster.local`.
|
||||
- Expose `/healthz`, `/identity`, and `/check` on port `8080`.
|
||||
- Keep `/healthz` local and fast; it must not perform DNS or peer requests.
|
||||
- Run peer checks only when `/check` is requested and report source/peer pod and
|
||||
node names.
|
||||
- Report `all_reachable: true` only when every peer responds successfully.
|
||||
- Report `all_cross_node: true` only when every successful peer response comes
|
||||
from a different node.
|
||||
- Support HTTPS through the mounted `k3s-connectivity-tls` Secret.
|
||||
- Pull the multi-architecture registry image for `amd64`, `arm64`, and
|
||||
`arm/v7` nodes.
|
||||
- Leave pods Pending rather than co-locating them when insufficient eligible
|
||||
nodes are available; nodes with a disk-pressure `NoSchedule` taint are not
|
||||
eligible.
|
||||
Binary file not shown.
Binary file not shown.
+50
-118
@@ -1,15 +1,12 @@
|
||||
#!/usr/bin/env python3
|
||||
"""HTTP/HTTPS connectivity probe for StatefulSet pods."""
|
||||
"""Simple HTTP endpoint and on-demand peer connectivity checker."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import socket
|
||||
import ssl
|
||||
import threading
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from dataclasses import asdict, dataclass
|
||||
from datetime import datetime, timezone
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
@@ -20,21 +17,20 @@ from urllib.parse import urlsplit
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
LOG = logging.getLogger("k3s-connectivity")
|
||||
CLUSTER_DOMAIN = "cluster.local"
|
||||
TLS_CERT_FILE = "/tls/tls.crt"
|
||||
TLS_KEY_FILE = "/tls/tls.key"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Settings:
|
||||
app_name: str
|
||||
namespace: str
|
||||
cluster_domain: str
|
||||
replicas: int
|
||||
protocol: str
|
||||
port: int
|
||||
request_timeout_seconds: float
|
||||
probe_interval_seconds: float
|
||||
verify_tls: bool
|
||||
tls_cert_file: str
|
||||
tls_key_file: str
|
||||
|
||||
@classmethod
|
||||
def from_env(cls, environment: Mapping[str, str] | None = None) -> "Settings":
|
||||
@@ -46,31 +42,25 @@ class Settings:
|
||||
replicas = int(environment.get("REPLICAS", "2"))
|
||||
port = int(environment.get("PORT", "8080"))
|
||||
timeout = float(environment.get("REQUEST_TIMEOUT_SECONDS", "5"))
|
||||
interval = float(environment.get("PROBE_INTERVAL_SECONDS", "30"))
|
||||
if replicas < 1 or port < 1 or timeout <= 0 or interval < 0:
|
||||
raise ValueError("invalid replicas, port, timeout, or probe interval")
|
||||
if replicas < 1 or port < 1 or timeout <= 0:
|
||||
raise ValueError("invalid replicas, port, or request timeout")
|
||||
|
||||
return cls(
|
||||
app_name=environment.get("APP_NAME", "k3s-connectivity"),
|
||||
namespace=environment.get("POD_NAMESPACE", "default"),
|
||||
cluster_domain=environment.get("CLUSTER_DOMAIN", "cluster.local"),
|
||||
replicas=replicas,
|
||||
protocol=protocol,
|
||||
port=port,
|
||||
request_timeout_seconds=timeout,
|
||||
probe_interval_seconds=interval,
|
||||
verify_tls=environment.get("VERIFY_TLS", "false").lower()
|
||||
in {"1", "true", "yes"},
|
||||
tls_cert_file=environment.get("TLS_CERT_FILE", "/tls/tls.crt"),
|
||||
tls_key_file=environment.get("TLS_KEY_FILE", "/tls/tls.key"),
|
||||
)
|
||||
|
||||
def peer_targets(self, hostname: str) -> list["PeerTarget"]:
|
||||
local_name = hostname.split(".", maxsplit=1)[0]
|
||||
suffix = f"{self.app_name}.{self.namespace}.svc.{self.cluster_domain}"
|
||||
suffix = f"{self.app_name}.{self.namespace}.svc.{CLUSTER_DOMAIN}"
|
||||
return [
|
||||
PeerTarget(
|
||||
ordinal=ordinal,
|
||||
name=f"{self.app_name}-{ordinal}",
|
||||
host=f"{self.app_name}-{ordinal}.{suffix}",
|
||||
)
|
||||
@@ -81,7 +71,6 @@ class Settings:
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PeerTarget:
|
||||
ordinal: int
|
||||
name: str
|
||||
host: str
|
||||
|
||||
@@ -101,20 +90,6 @@ class PodIdentity:
|
||||
)
|
||||
|
||||
|
||||
class ProbeStore:
|
||||
def __init__(self) -> None:
|
||||
self._lock = threading.Lock()
|
||||
self._latest: dict[str, Any] | None = None
|
||||
|
||||
def replace(self, report: dict[str, Any]) -> None:
|
||||
with self._lock:
|
||||
self._latest = report
|
||||
|
||||
def latest(self) -> dict[str, Any]:
|
||||
with self._lock:
|
||||
return self._latest or {"state": "not-run"}
|
||||
|
||||
|
||||
def _timestamp() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
@@ -127,41 +102,35 @@ def _ssl_context(settings: Settings) -> ssl.SSLContext | None:
|
||||
|
||||
def probe_peer(target: PeerTarget, settings: Settings,
|
||||
source: PodIdentity) -> dict[str, Any]:
|
||||
url = f"{settings.protocol}://{target.host}:{settings.port}/healthz"
|
||||
checked_at = _timestamp()
|
||||
url = f"{settings.protocol}://{target.host}:{settings.port}/identity"
|
||||
request = Request(url, headers={"Accept": "application/json"})
|
||||
try:
|
||||
with urlopen(request, timeout=settings.request_timeout_seconds,
|
||||
context=_ssl_context(settings)) as response:
|
||||
payload = json.loads(response.read().decode("utf-8"))
|
||||
peer = payload.get("pod", {})
|
||||
peer_node = peer.get("node_name", "")
|
||||
return {
|
||||
"target": target.name,
|
||||
"url": url,
|
||||
"checked_at": checked_at,
|
||||
"success": 200 <= response.status < 300,
|
||||
"status": response.status,
|
||||
"peer_hostname": peer.get("hostname", ""),
|
||||
"peer_node": peer_node,
|
||||
"cross_node": bool(source.node_name and peer_node)
|
||||
and source.node_name != peer_node,
|
||||
"detail": "ok",
|
||||
}
|
||||
peer = payload.get("pod", {})
|
||||
peer_node = peer.get("node_name", "")
|
||||
return {
|
||||
"target": target.name,
|
||||
"url": url,
|
||||
"reachable": 200 <= response.status < 300,
|
||||
"peer_hostname": peer.get("hostname", ""),
|
||||
"peer_node": peer_node,
|
||||
"cross_node": bool(source.node_name and peer_node)
|
||||
and source.node_name != peer_node,
|
||||
"detail": "ok",
|
||||
}
|
||||
except HTTPError as error:
|
||||
return _failed_result(target, url, checked_at, f"HTTP {error.code}")
|
||||
return _failed_result(target, url, f"HTTP {error.code}")
|
||||
except (URLError, OSError, TimeoutError, ValueError) as error:
|
||||
return _failed_result(target, url, checked_at, str(error))
|
||||
return _failed_result(target, url, str(error))
|
||||
|
||||
|
||||
def _failed_result(target: PeerTarget, url: str, checked_at: str,
|
||||
detail: str) -> dict[str, Any]:
|
||||
def _failed_result(target: PeerTarget, url: str, detail: str) -> dict[str, Any]:
|
||||
return {
|
||||
"target": target.name,
|
||||
"url": url,
|
||||
"checked_at": checked_at,
|
||||
"success": False,
|
||||
"status": None,
|
||||
"reachable": False,
|
||||
"peer_hostname": "",
|
||||
"peer_node": "",
|
||||
"cross_node": False,
|
||||
@@ -169,52 +138,43 @@ def _failed_result(target: PeerTarget, url: str, checked_at: str,
|
||||
}
|
||||
|
||||
|
||||
class ProbeApplication:
|
||||
class ConnectivityApplication:
|
||||
def __init__(self, settings: Settings, identity: PodIdentity) -> None:
|
||||
self.settings = settings
|
||||
self.identity = identity
|
||||
self.store = ProbeStore()
|
||||
|
||||
def health(self) -> dict[str, Any]:
|
||||
return {"status": "ok", "pod": asdict(self.identity)}
|
||||
def health(self) -> dict[str, str]:
|
||||
return {"status": "ok"}
|
||||
|
||||
def probe_now(self) -> dict[str, Any]:
|
||||
targets = self.settings.peer_targets(self.identity.hostname)
|
||||
results: list[dict[str, Any]] = []
|
||||
if targets:
|
||||
with ThreadPoolExecutor(max_workers=min(len(targets), 8)) as executor:
|
||||
futures = {
|
||||
executor.submit(probe_peer, target, self.settings, self.identity): target
|
||||
for target in targets
|
||||
}
|
||||
for future in as_completed(futures):
|
||||
results.append(future.result())
|
||||
results.sort(key=lambda result: result["target"])
|
||||
report = {
|
||||
def identity_response(self) -> dict[str, dict[str, str]]:
|
||||
return {"pod": asdict(self.identity)}
|
||||
|
||||
def check_peers(self) -> dict[str, Any]:
|
||||
peers = [
|
||||
probe_peer(target, self.settings, self.identity)
|
||||
for target in self.settings.peer_targets(self.identity.hostname)
|
||||
]
|
||||
return {
|
||||
"checked_at": _timestamp(),
|
||||
"source": asdict(self.identity),
|
||||
"protocol": self.settings.protocol,
|
||||
"all_reachable": all(result["success"] for result in results),
|
||||
"all_cross_node": all(result["cross_node"] for result in results),
|
||||
"peers": results,
|
||||
"all_reachable": all(peer["reachable"] for peer in peers),
|
||||
"all_cross_node": all(peer["cross_node"] for peer in peers),
|
||||
"peers": peers,
|
||||
}
|
||||
self.store.replace(report)
|
||||
return report
|
||||
|
||||
|
||||
class ConnectivityRequestHandler(BaseHTTPRequestHandler):
|
||||
server: ThreadingHTTPServer
|
||||
|
||||
def do_GET(self) -> None:
|
||||
application: ProbeApplication = getattr(self.server, "application")
|
||||
application: ConnectivityApplication = getattr(self.server, "application")
|
||||
path = urlsplit(self.path).path
|
||||
if path in {"/", "/healthz"}:
|
||||
self._send_json(200, application.health())
|
||||
elif path == "/probe":
|
||||
report = application.probe_now()
|
||||
self._send_json(200 if report["all_reachable"] else 503, report)
|
||||
elif path == "/results":
|
||||
self._send_json(200, application.store.latest())
|
||||
elif path == "/identity":
|
||||
self._send_json(200, application.identity_response())
|
||||
elif path == "/check":
|
||||
self._send_json(200, application.check_peers())
|
||||
else:
|
||||
self._send_json(404, {"error": "not found"})
|
||||
|
||||
@@ -237,49 +197,22 @@ class ConnectivityRequestHandler(BaseHTTPRequestHandler):
|
||||
def _configure_tls(server: ThreadingHTTPServer, settings: Settings) -> None:
|
||||
if settings.protocol != "https":
|
||||
return
|
||||
if not Path(settings.tls_cert_file).is_file() or not Path(settings.tls_key_file).is_file():
|
||||
raise FileNotFoundError("TLS_CERT_FILE and TLS_KEY_FILE must reference mounted files")
|
||||
if not Path(TLS_CERT_FILE).is_file() or not Path(TLS_KEY_FILE).is_file():
|
||||
raise FileNotFoundError("mounted TLS certificate and key are missing")
|
||||
context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
|
||||
context.load_cert_chain(settings.tls_cert_file, settings.tls_key_file)
|
||||
context.load_cert_chain(TLS_CERT_FILE, TLS_KEY_FILE)
|
||||
server.socket = context.wrap_socket(server.socket, server_side=True)
|
||||
|
||||
|
||||
def _run_periodic_probes(application: ProbeApplication,
|
||||
stop_event: threading.Event) -> None:
|
||||
while not stop_event.is_set():
|
||||
report = application.probe_now()
|
||||
LOG.info("probe complete reachable=%s cross_node=%s peers=%d",
|
||||
report["all_reachable"], report["all_cross_node"],
|
||||
len(report["peers"]))
|
||||
if application.settings.probe_interval_seconds == 0:
|
||||
return
|
||||
stop_event.wait(application.settings.probe_interval_seconds)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--once", action="store_true")
|
||||
arguments = parser.parse_args()
|
||||
|
||||
logging.basicConfig(level=os.environ.get("LOG_LEVEL", "INFO"),
|
||||
logging.basicConfig(level=logging.INFO,
|
||||
format="%(asctime)s %(levelname)s %(message)s")
|
||||
settings = Settings.from_env()
|
||||
application = ProbeApplication(settings, PodIdentity.local())
|
||||
if arguments.once:
|
||||
report = application.probe_now()
|
||||
print(json.dumps(report, sort_keys=True))
|
||||
return 0 if report["all_reachable"] else 1
|
||||
|
||||
application = ConnectivityApplication(settings, PodIdentity.local())
|
||||
server = ThreadingHTTPServer(("", settings.port), ConnectivityRequestHandler)
|
||||
server.daemon_threads = True
|
||||
setattr(server, "application", application)
|
||||
_configure_tls(server, settings)
|
||||
stop_event = threading.Event()
|
||||
probe_thread = threading.Thread(
|
||||
target=_run_periodic_probes,
|
||||
args=(application, stop_event),
|
||||
daemon=True,
|
||||
)
|
||||
probe_thread.start()
|
||||
try:
|
||||
LOG.info("listening protocol=%s port=%d pod=%s node=%s",
|
||||
settings.protocol, settings.port, application.identity.hostname,
|
||||
@@ -288,7 +221,6 @@ def main() -> int:
|
||||
except KeyboardInterrupt:
|
||||
return 0
|
||||
finally:
|
||||
stop_event.set()
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
|
||||
|
||||
+3
-4
@@ -30,7 +30,7 @@ metadata:
|
||||
app.kubernetes.io/name: k3s-connectivity
|
||||
spec:
|
||||
serviceName: k3s-connectivity
|
||||
replicas: 2
|
||||
replicas: 3
|
||||
podManagementPolicy: Parallel
|
||||
selector:
|
||||
matchLabels:
|
||||
@@ -80,13 +80,11 @@ spec:
|
||||
fieldRef:
|
||||
fieldPath: status.podIP
|
||||
- name: REPLICAS
|
||||
value: "2"
|
||||
value: "3"
|
||||
- name: PROTOCOL
|
||||
value: http
|
||||
- name: PORT
|
||||
value: "8080"
|
||||
- name: PROBE_INTERVAL_SECONDS
|
||||
value: "30"
|
||||
- name: REQUEST_TIMEOUT_SECONDS
|
||||
value: "5"
|
||||
readinessProbe:
|
||||
@@ -113,6 +111,7 @@ spec:
|
||||
port: web
|
||||
scheme: HTTP
|
||||
periodSeconds: 2
|
||||
timeoutSeconds: 2
|
||||
failureThreshold: 15
|
||||
resources:
|
||||
requests:
|
||||
|
||||
+42
-1
@@ -1,6 +1,11 @@
|
||||
import json
|
||||
import threading
|
||||
import unittest
|
||||
from http.server import ThreadingHTTPServer
|
||||
from urllib.request import urlopen
|
||||
|
||||
from app import Settings
|
||||
from app import (ConnectivityApplication, ConnectivityRequestHandler,
|
||||
PodIdentity, Settings)
|
||||
|
||||
|
||||
class SettingsTests(unittest.TestCase):
|
||||
@@ -27,5 +32,41 @@ class SettingsTests(unittest.TestCase):
|
||||
Settings.from_env({"PROTOCOL": "tcp"})
|
||||
|
||||
|
||||
class EndpointTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
settings = Settings.from_env({"REPLICAS": "1", "PORT": "8080"})
|
||||
application = ConnectivityApplication(
|
||||
settings,
|
||||
PodIdentity("k3s-connectivity-0", "test-node", "127.0.0.1"),
|
||||
)
|
||||
self.server = ThreadingHTTPServer(
|
||||
("127.0.0.1", 0), ConnectivityRequestHandler)
|
||||
self.server.daemon_threads = True
|
||||
setattr(self.server, "application", application)
|
||||
self.thread = threading.Thread(target=self.server.serve_forever)
|
||||
self.thread.start()
|
||||
self.base_url = f"http://127.0.0.1:{self.server.server_port}"
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self.server.shutdown()
|
||||
self.server.server_close()
|
||||
self.thread.join()
|
||||
|
||||
def test_health_is_local_and_fast(self) -> None:
|
||||
with urlopen(f"{self.base_url}/healthz", timeout=1) as response:
|
||||
payload = json.loads(response.read())
|
||||
|
||||
self.assertEqual(response.status, 200)
|
||||
self.assertEqual(payload, {"status": "ok"})
|
||||
|
||||
def test_check_succeeds_without_peers(self) -> None:
|
||||
with urlopen(f"{self.base_url}/check", timeout=1) as response:
|
||||
payload = json.loads(response.read())
|
||||
|
||||
self.assertTrue(payload["all_reachable"])
|
||||
self.assertTrue(payload["all_cross_node"])
|
||||
self.assertEqual(payload["peers"], [])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user