This commit is contained in:
2026-10-16 18:28:59 +02:00
parent b728807c5f
commit 7fbaf5c94a
17 changed files with 688 additions and 79 deletions
+21
View File
@@ -0,0 +1,21 @@
name: Build K3s Connectivity Image
on:
push:
paths:
- k3s/**
- .gitea/workflows/build-k3s-image.yaml
pull_request:
paths:
- k3s/**
- .gitea/workflows/build-k3s-image.yaml
workflow_dispatch:
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Check out source
uses: actions/checkout@v4
- name: Build image
run: docker build --pull --tag k3s-connectivity-tester:${{ gitea.sha }} ./k3s
-49
View File
@@ -1,49 +0,0 @@
name: release
on:
push:
branches:
- 'auto-*'
tags:
- 'v*'
workflow_dispatch:
inputs:
tag:
description: 'Image tag (e.g. auto-<sha>)'
required: false
default: latest
type: string
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: checkout
uses: actions/checkout@v4
- name: log into Gitea Packages
uses: docker/login-action@v3
with:
registry: gitea.rudolf.keenetic.link
username: ${{ secrets.REGISTRY_USERNAME }}
password: ${{ secrets.REGISTRY_TOKEN }}
- name: resolve version
id: version
run: |
if [ -n "${{ inputs.tag }}" ] && [ "${{ inputs.tag }}" != "latest" ]; then
echo "VERSION=${{ inputs.tag }}" >> "$GITHUB_OUTPUT"
else
echo "VERSION=${GITHUB_REF_NAME##*/}" >> "$GITHUB_OUTPUT"
fi
echo "VERSION=$VERSION"
- name: build and push
uses: docker/build-push-action@v6
with:
context: .
file: Dockerfile
push: true
tags: |
gitea.rudolf.keenetic.link/org1/swarm_connectivity_tester:${{ steps.version.outputs.VERSION }}
gitea.rudolf.keenetic.link/org1/swarm_connectivity_tester:latest
-12
View File
@@ -1,12 +0,0 @@
FROM python:3.12-slim
WORKDIR /app
COPY . /app/
RUN pip install --no-cache-dir docker==7.1.0
ENV PYTHONUNBUFFERED=1 \
PERIOD_SECONDS=600 \
CHECK_TIMEOUT_S=3
CMD ["python", "-m", "app"]
Binary file not shown.
Binary file not shown.
+51 -13
View File
@@ -11,16 +11,15 @@ each round a final loop checks PUBLIC_SERVICES. Cross-validation:
from __future__ import annotations
import asyncio
import argparse
import importlib
import logging
import os
import socket
import subprocess
import sys
import time
from typing import Iterable, List, Set, Tuple
import docker
from docker.models.containers import Container
from docker.models.networks import Network
from typing import Any, Iterable, List, Set, Tuple
from network_map import Endpoint, GLOBAL_SERVICES, NEGATIVE, NETWORK_MAP, PUBLIC_SERVICES
@@ -34,7 +33,7 @@ def _socket_path() -> str:
return os.environ.get("DOCKER_SOCKET", "/var/run/docker.sock")
def _self_container(client: docker.DockerClient) -> Container:
def _self_container(client: Any) -> Any:
hostname = socket.gethostname()
for c in client.containers.list():
cn = c.name.lstrip("/")
@@ -47,7 +46,7 @@ def _self_container(client: docker.DockerClient) -> Container:
raise RuntimeError(f"self container not found for hostname={hostname}")
def _swarm_networks_for(client: docker.DockerClient,
def _swarm_networks_for(client: Any,
names: Iterable[str]) -> dict:
out = {}
by_name = {n.name: n for n in client.networks.list()}
@@ -58,8 +57,7 @@ def _swarm_networks_for(client: docker.DockerClient,
return out
def _attach(client: docker.DockerClient, container: Container,
network: Network) -> bool:
def _attach(client: Any, container: Any, network: Any) -> bool:
current = (container.attrs.get("NetworkSettings", {}) or {}).get("Networks", {})
if network.name in current:
return False
@@ -73,8 +71,7 @@ def _attach(client: docker.DockerClient, container: Container,
raise
def _detach(client: docker.DockerClient, container: Container,
network: Network) -> None:
def _detach(client: Any, container: Any, network: Any) -> None:
try:
network.disconnect(container, force=True)
except Exception as exc:
@@ -214,7 +211,7 @@ async def _check_negative(endpoints: List[Endpoint],
return leaks
async def main() -> int:
async def run_checks() -> int:
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(levelname)s %(message)s",
@@ -224,6 +221,7 @@ async def main() -> int:
timeout = float(os.environ.get("CHECK_TIMEOUT_S", "3"))
LOG.info("starting; period=%ds timeout=%.1fs", period, timeout)
docker = importlib.import_module("docker")
client = docker.DockerClient(base_url="unix://" + _socket_path())
self_container = _self_container(client)
@@ -272,5 +270,45 @@ async def main() -> int:
await asyncio.sleep(sleep_for)
def _healthcheck() -> int:
try:
with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as docker_socket:
docker_socket.connect(_socket_path())
except OSError as exc:
print(f"Docker socket is unavailable: {exc}", file=sys.stderr)
return 1
return 0
def _entrypoint() -> int:
socket_path = _socket_path()
if not os.path.exists(socket_path):
print(f"connectivity-tester: Docker socket is missing: {socket_path}",
file=sys.stderr)
return 1
subprocess.run(
[sys.executable, "-m", "pip", "install", "--disable-pip-version-check",
"--no-cache-dir", "--requirement", "/app/requirements.txt"],
check=True,
)
os.execv(sys.executable, [sys.executable, __file__])
return 0
def main() -> int:
parser = argparse.ArgumentParser()
modes = parser.add_mutually_exclusive_group()
modes.add_argument("--entrypoint", action="store_true")
modes.add_argument("--healthcheck", action="store_true")
args = parser.parse_args()
if args.entrypoint:
return _entrypoint()
if args.healthcheck:
return _healthcheck()
return asyncio.run(run_checks())
if __name__ == "__main__":
asyncio.run(main())
raise SystemExit(main())
+10
View File
@@ -0,0 +1,10 @@
FROM python:3.12-alpine
WORKDIR /app
COPY app.py /app/app.py
ENV PYTHONDONTWRITEBYTECODE=1
USER 65532:65532
EXPOSE 8080
ENTRYPOINT ["python", "/app/app.py"]
+85
View File
@@ -0,0 +1,85 @@
# K3s Cross-Node Connectivity Tester
This workload checks HTTP connectivity between StatefulSet pods by their
individual internal DNS names. A successful `/probe` response includes the
source and peer node names and reports `all_cross_node: true` only when every
peer responded from a different node.
The default deployment uses two replicas and HTTP on port `8080`.
## Build and distribute the image
Build the image from this directory:
```sh
docker build -t k3s-connectivity-tester:0.1.0 .
```
For a multi-node K3s cluster, publish that image to a registry reachable by
every node, then change `image:` in `kubernetes.yaml` to its full registry
name and set `imagePullPolicy: Always`. Alternatively, import the local image
into the K3s containerd image store on every node:
```sh
docker save k3s-connectivity-tester:0.1.0 | sudo k3s ctr images import -
```
## Deploy and verify HTTP
```sh
kubectl apply -k .
kubectl -n connectivity-test rollout status statefulset/k3s-connectivity
kubectl -n connectivity-test get pods -o wide
```
The two pods must show different values in the `NODE` column. This is enforced
by required pod anti-affinity and a `DoNotSchedule` hostname spread rule. On a
single-node cluster, the second pod deliberately remains Pending rather than
silently testing same-node traffic.
Run an immediate probe from the first pod:
```sh
kubectl -n connectivity-test exec k3s-connectivity-0 -- \
python -c "from urllib.request import urlopen; print(urlopen('http://localhost:8080/probe').read().decode())"
```
The response contains a peer URL such as:
```text
http://k3s-connectivity-1.k3s-connectivity.connectivity-test.svc.cluster.local:8080/healthz
```
Check the latest periodic result without triggering another request:
```sh
kubectl -n connectivity-test exec k3s-connectivity-0 -- \
python -c "from urllib.request import urlopen; print(urlopen('http://localhost:8080/results').read().decode())"
```
`all_reachable` and `all_cross_node` must both be `true`. The response also
includes the exact source and peer node names, which makes failed placement or
network-policy diagnosis straightforward.
## Enable HTTPS
Create a TLS Secret in the workload namespace, then apply the included
strategic merge patch:
```sh
kubectl -n connectivity-test create secret tls k3s-connectivity-tls \
--cert=tls.crt --key=tls.key
kubectl -n connectivity-test patch statefulset k3s-connectivity \
--type=strategic --patch-file=https-patch.yaml
kubectl -n connectivity-test rollout status statefulset/k3s-connectivity
```
The HTTPS patch disables certificate verification for a self-signed diagnostic
certificate. For a certificate issued by a trusted internal CA with SANs for
the pod DNS names, set `VERIFY_TLS` to `true` in the StatefulSet.
To remove the test:
```sh
kubectl delete -k .
```
Binary file not shown.
Binary file not shown.
+297
View File
@@ -0,0 +1,297 @@
#!/usr/bin/env python3
"""HTTP/HTTPS connectivity probe for StatefulSet pods."""
from __future__ import annotations
import argparse
import json
import logging
import os
import socket
import ssl
import threading
from concurrent.futures import ThreadPoolExecutor, as_completed
from dataclasses import asdict, dataclass
from datetime import datetime, timezone
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from typing import Any, Mapping
from urllib.error import HTTPError, URLError
from urllib.parse import urlsplit
from urllib.request import Request, urlopen
LOG = logging.getLogger("k3s-connectivity")
@dataclass(frozen=True)
class Settings:
app_name: str
namespace: str
cluster_domain: str
replicas: int
protocol: str
port: int
request_timeout_seconds: float
probe_interval_seconds: float
verify_tls: bool
tls_cert_file: str
tls_key_file: str
@classmethod
def from_env(cls, environment: Mapping[str, str] | None = None) -> "Settings":
environment = environment or os.environ
protocol = environment.get("PROTOCOL", "http").lower()
if protocol not in {"http", "https"}:
raise ValueError("PROTOCOL must be either http or https")
replicas = int(environment.get("REPLICAS", "2"))
port = int(environment.get("PORT", "8080"))
timeout = float(environment.get("REQUEST_TIMEOUT_SECONDS", "5"))
interval = float(environment.get("PROBE_INTERVAL_SECONDS", "30"))
if replicas < 1 or port < 1 or timeout <= 0 or interval < 0:
raise ValueError("invalid replicas, port, timeout, or probe interval")
return cls(
app_name=environment.get("APP_NAME", "k3s-connectivity"),
namespace=environment.get("POD_NAMESPACE", "default"),
cluster_domain=environment.get("CLUSTER_DOMAIN", "cluster.local"),
replicas=replicas,
protocol=protocol,
port=port,
request_timeout_seconds=timeout,
probe_interval_seconds=interval,
verify_tls=environment.get("VERIFY_TLS", "false").lower()
in {"1", "true", "yes"},
tls_cert_file=environment.get("TLS_CERT_FILE", "/tls/tls.crt"),
tls_key_file=environment.get("TLS_KEY_FILE", "/tls/tls.key"),
)
def peer_targets(self, hostname: str) -> list["PeerTarget"]:
local_name = hostname.split(".", maxsplit=1)[0]
suffix = f"{self.app_name}.{self.namespace}.svc.{self.cluster_domain}"
return [
PeerTarget(
ordinal=ordinal,
name=f"{self.app_name}-{ordinal}",
host=f"{self.app_name}-{ordinal}.{suffix}",
)
for ordinal in range(self.replicas)
if f"{self.app_name}-{ordinal}" != local_name
]
@dataclass(frozen=True)
class PeerTarget:
ordinal: int
name: str
host: str
@dataclass(frozen=True)
class PodIdentity:
hostname: str
node_name: str
pod_ip: str
@classmethod
def local(cls) -> "PodIdentity":
return cls(
hostname=socket.gethostname(),
node_name=os.environ.get("NODE_NAME", ""),
pod_ip=os.environ.get("POD_IP", ""),
)
class ProbeStore:
def __init__(self) -> None:
self._lock = threading.Lock()
self._latest: dict[str, Any] | None = None
def replace(self, report: dict[str, Any]) -> None:
with self._lock:
self._latest = report
def latest(self) -> dict[str, Any]:
with self._lock:
return self._latest or {"state": "not-run"}
def _timestamp() -> str:
return datetime.now(timezone.utc).isoformat()
def _ssl_context(settings: Settings) -> ssl.SSLContext | None:
if settings.protocol != "https" or settings.verify_tls:
return None
return ssl._create_unverified_context()
def probe_peer(target: PeerTarget, settings: Settings,
source: PodIdentity) -> dict[str, Any]:
url = f"{settings.protocol}://{target.host}:{settings.port}/healthz"
checked_at = _timestamp()
request = Request(url, headers={"Accept": "application/json"})
try:
with urlopen(request, timeout=settings.request_timeout_seconds,
context=_ssl_context(settings)) as response:
payload = json.loads(response.read().decode("utf-8"))
peer = payload.get("pod", {})
peer_node = peer.get("node_name", "")
return {
"target": target.name,
"url": url,
"checked_at": checked_at,
"success": 200 <= response.status < 300,
"status": response.status,
"peer_hostname": peer.get("hostname", ""),
"peer_node": peer_node,
"cross_node": bool(source.node_name and peer_node)
and source.node_name != peer_node,
"detail": "ok",
}
except HTTPError as error:
return _failed_result(target, url, checked_at, f"HTTP {error.code}")
except (URLError, OSError, TimeoutError, ValueError) as error:
return _failed_result(target, url, checked_at, str(error))
def _failed_result(target: PeerTarget, url: str, checked_at: str,
detail: str) -> dict[str, Any]:
return {
"target": target.name,
"url": url,
"checked_at": checked_at,
"success": False,
"status": None,
"peer_hostname": "",
"peer_node": "",
"cross_node": False,
"detail": detail,
}
class ProbeApplication:
def __init__(self, settings: Settings, identity: PodIdentity) -> None:
self.settings = settings
self.identity = identity
self.store = ProbeStore()
def health(self) -> dict[str, Any]:
return {"status": "ok", "pod": asdict(self.identity)}
def probe_now(self) -> dict[str, Any]:
targets = self.settings.peer_targets(self.identity.hostname)
results: list[dict[str, Any]] = []
if targets:
with ThreadPoolExecutor(max_workers=min(len(targets), 8)) as executor:
futures = {
executor.submit(probe_peer, target, self.settings, self.identity): target
for target in targets
}
for future in as_completed(futures):
results.append(future.result())
results.sort(key=lambda result: result["target"])
report = {
"checked_at": _timestamp(),
"source": asdict(self.identity),
"protocol": self.settings.protocol,
"all_reachable": all(result["success"] for result in results),
"all_cross_node": all(result["cross_node"] for result in results),
"peers": results,
}
self.store.replace(report)
return report
class ConnectivityRequestHandler(BaseHTTPRequestHandler):
server: ThreadingHTTPServer
def do_GET(self) -> None:
application: ProbeApplication = getattr(self.server, "application")
path = urlsplit(self.path).path
if path in {"/", "/healthz"}:
self._send_json(200, application.health())
elif path == "/probe":
report = application.probe_now()
self._send_json(200 if report["all_reachable"] else 503, report)
elif path == "/results":
self._send_json(200, application.store.latest())
else:
self._send_json(404, {"error": "not found"})
def do_HEAD(self) -> None:
self.do_GET()
def _send_json(self, status: int, payload: dict[str, Any]) -> None:
body = json.dumps(payload, sort_keys=True).encode("utf-8")
self.send_response(status)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
if self.command != "HEAD":
self.wfile.write(body)
def log_message(self, format_string: str, *arguments: object) -> None:
LOG.info("%s - %s", self.address_string(), format_string % arguments)
def _configure_tls(server: ThreadingHTTPServer, settings: Settings) -> None:
if settings.protocol != "https":
return
if not Path(settings.tls_cert_file).is_file() or not Path(settings.tls_key_file).is_file():
raise FileNotFoundError("TLS_CERT_FILE and TLS_KEY_FILE must reference mounted files")
context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
context.load_cert_chain(settings.tls_cert_file, settings.tls_key_file)
server.socket = context.wrap_socket(server.socket, server_side=True)
def _run_periodic_probes(application: ProbeApplication,
stop_event: threading.Event) -> None:
while not stop_event.is_set():
report = application.probe_now()
LOG.info("probe complete reachable=%s cross_node=%s peers=%d",
report["all_reachable"], report["all_cross_node"],
len(report["peers"]))
if application.settings.probe_interval_seconds == 0:
return
stop_event.wait(application.settings.probe_interval_seconds)
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--once", action="store_true")
arguments = parser.parse_args()
logging.basicConfig(level=os.environ.get("LOG_LEVEL", "INFO"),
format="%(asctime)s %(levelname)s %(message)s")
settings = Settings.from_env()
application = ProbeApplication(settings, PodIdentity.local())
if arguments.once:
report = application.probe_now()
print(json.dumps(report, sort_keys=True))
return 0 if report["all_reachable"] else 1
server = ThreadingHTTPServer(("", settings.port), ConnectivityRequestHandler)
setattr(server, "application", application)
_configure_tls(server, settings)
stop_event = threading.Event()
probe_thread = threading.Thread(
target=_run_periodic_probes,
args=(application, stop_event),
daemon=True,
)
probe_thread.start()
try:
LOG.info("listening protocol=%s port=%d pod=%s node=%s",
settings.protocol, settings.port, application.identity.hostname,
application.identity.node_name)
server.serve_forever()
except KeyboardInterrupt:
return 0
finally:
stop_event.set()
server.shutdown()
server.server_close()
if __name__ == "__main__":
raise SystemExit(main())
+32
View File
@@ -0,0 +1,32 @@
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: k3s-connectivity
namespace: connectivity-test
spec:
template:
spec:
containers:
- name: connectivity-tester
env:
- name: PROTOCOL
value: https
- name: VERIFY_TLS
value: "false"
volumeMounts:
- name: tls
mountPath: /tls
readOnly: true
readinessProbe:
httpGet:
scheme: HTTPS
livenessProbe:
httpGet:
scheme: HTTPS
startupProbe:
httpGet:
scheme: HTTPS
volumes:
- name: tls
secret:
secretName: k3s-connectivity-tls
+134
View File
@@ -0,0 +1,134 @@
apiVersion: v1
kind: Namespace
metadata:
name: connectivity-test
---
apiVersion: v1
kind: Service
metadata:
name: k3s-connectivity
namespace: connectivity-test
labels:
app.kubernetes.io/name: k3s-connectivity
spec:
clusterIP: None
publishNotReadyAddresses: true
selector:
app.kubernetes.io/name: k3s-connectivity
ports:
- name: web
port: 8080
targetPort: web
protocol: TCP
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: k3s-connectivity
namespace: connectivity-test
labels:
app.kubernetes.io/name: k3s-connectivity
spec:
serviceName: k3s-connectivity
replicas: 2
podManagementPolicy: Parallel
selector:
matchLabels:
app.kubernetes.io/name: k3s-connectivity
template:
metadata:
labels:
app.kubernetes.io/name: k3s-connectivity
spec:
automountServiceAccountToken: false
terminationGracePeriodSeconds: 10
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchLabels:
app.kubernetes.io/name: k3s-connectivity
topologyKey: kubernetes.io/hostname
topologySpreadConstraints:
- maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: DoNotSchedule
labelSelector:
matchLabels:
app.kubernetes.io/name: k3s-connectivity
containers:
- name: connectivity-tester
image: k3s-connectivity-tester:0.1.0
imagePullPolicy: IfNotPresent
ports:
- name: web
containerPort: 8080
protocol: TCP
env:
- name: APP_NAME
value: k3s-connectivity
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
- name: REPLICAS
value: "2"
- name: PROTOCOL
value: http
- name: PORT
value: "8080"
- name: PROBE_INTERVAL_SECONDS
value: "30"
- name: REQUEST_TIMEOUT_SECONDS
value: "5"
readinessProbe:
httpGet:
path: /healthz
port: web
scheme: HTTP
initialDelaySeconds: 2
periodSeconds: 5
timeoutSeconds: 2
failureThreshold: 3
livenessProbe:
httpGet:
path: /healthz
port: web
scheme: HTTP
initialDelaySeconds: 5
periodSeconds: 10
timeoutSeconds: 2
failureThreshold: 3
startupProbe:
httpGet:
path: /healthz
port: web
scheme: HTTP
periodSeconds: 2
failureThreshold: 15
resources:
requests:
cpu: 25m
memory: 32Mi
limits:
cpu: 100m
memory: 128Mi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 65532
runAsGroup: 65532
seccompProfile:
type: RuntimeDefault
+4
View File
@@ -0,0 +1,4 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- kubernetes.yaml
+31
View File
@@ -0,0 +1,31 @@
import unittest
from app import Settings
class SettingsTests(unittest.TestCase):
def test_peer_targets_use_stable_statefulset_names(self) -> None:
settings = Settings.from_env({
"APP_NAME": "k3s-connectivity",
"POD_NAMESPACE": "connectivity-test",
"CLUSTER_DOMAIN": "cluster.local",
"REPLICAS": "3",
})
targets = settings.peer_targets("k3s-connectivity-1")
self.assertEqual([target.name for target in targets], [
"k3s-connectivity-0", "k3s-connectivity-2",
])
self.assertEqual(
targets[0].host,
"k3s-connectivity-0.k3s-connectivity.connectivity-test.svc.cluster.local",
)
def test_invalid_protocol_is_rejected(self) -> None:
with self.assertRaisesRegex(ValueError, "PROTOCOL"):
Settings.from_env({"PROTOCOL": "tcp"})
if __name__ == "__main__":
unittest.main()
+1
View File
@@ -0,0 +1 @@
docker==7.1.0
+22 -5
View File
@@ -1,19 +1,28 @@
services:
connectivity-tester:
build:
context: .
dockerfile: Dockerfile
image: gitea.rudolf.keenetic.link/Org1/swarm_connectivity_tester:${version}
image: docker.io/library/python:3.12-slim
hostname: connectivity-tester
user: "0:0"
working_dir: /app
entrypoint: ["python", "/app/app.py", "--entrypoint"]
volumes:
- /var/run/docker.sock:/var/run/docker.sock
healthcheck:
test: ["CMD-SHELL", "test -S /var/run/docker.sock"]
test: ["CMD", "python", "/app/app.py", "--healthcheck"]
interval: 60s
timeout: 5s
retries: 3
start_period: 30s
configs:
- source: connectivity_tester_app
target: /app/app.py
mode: 0444
- source: connectivity_tester_network_map
target: /app/network_map.py
mode: 0444
- source: connectivity_tester_requirements
target: /app/requirements.txt
mode: 0444
deploy:
mode: global
resources:
@@ -26,3 +35,11 @@ services:
condition: on-failure
delay: 10s
max_attempts: 3
configs:
connectivity_tester_app:
file: ./app.py
connectivity_tester_network_map:
file: ./network_map.py
connectivity_tester_requirements:
file: ./requirements.txt