From 0c95d8e1eee6b707d28dadc33ed1c52a7d9ba48e Mon Sep 17 00:00:00 2001 From: hermes Date: Sat, 29 Aug 2026 22:53:46 +0200 Subject: [PATCH 01/10] feat(hermes_squid): enable local STT via faster-whisper (runtime install) (#21) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What Wires up local STT (faster-whisper) into the hermes-gateway container so voice messages on Telegram transcribe without depending on external services. Closes #19. Supersedes the closed #20 (which used a custom Dockerfile-hermes; this version uses a bind-mounted entrypoint script instead since pypi.org is now whitelisted). ## Changes 1. **`hermes_squid/squid.conf`** — expands the egress whitelist: - new `stt_pkg_domains` for `pypi.org`, `files.pythonhosted.org`, `huggingface.co`, `github.com`, `models.dev`, `registry.npmjs.org`, `cdn-lfs.huggingface.co` — needed for runtime `pip install faster-whisper` and HF model fetch. - new `telegram_ips` covering AS62041 (91.108.4.0/22, 91.108.8.0/22, 91.108.12.0/22, 91.108.16.0/22, 91.108.20.0/22, 91.108.56.0/22, 149.154.160.0/20, 185.76.151.0/24) as a backstop for the `dstdomain` ACL. squid's `dstdomain` does not match HTTPS CONNECT requests without `ssl_bump`, so the recent log shows 24 of 9083 `api.telegram.org` requests getting 403 in a 5-minute window. - renamed `hermes_network` -> `localnet` (squid default name, no behavior change). 2. **`hermes_squid/hermes-gateway-entrypoint.sh`** (new) — bind-mounted into the container at `/opt/data/hermes-gateway-entrypoint.sh` and set as the service entrypoint. The script: - is idempotent: on first start, runs `apt-get install -y python3-pip ffmpeg` and `pip3 install --break-system-packages faster-whisper`. Subsequent restarts skip install if `faster_whisper` is already importable. - exports `STT_PROVIDER=local`, `STT_LOCAL_MODEL=base`, `STT_LOCAL_DEVICE=cpu`, `STT_LOCAL_COMPUTE_TYPE=int8`. - `exec`s the original gateway command (`gateway run`). 3. **`hermes_squid/stack-compose.yml`** — `hermes-gateway` gets: - `entrypoint: ["/opt/data/hermes-gateway-entrypoint.sh"]` - `command: ["gateway", "run"]` - bind-mount `./hermes-gateway-entrypoint.sh:/opt/data/hermes-gateway-entrypoint.sh:ro` - new env: `STT_PROVIDER`, `STT_LOCAL_MODEL`, `STT_LOCAL_DEVICE`, `STT_LOCAL_COMPUTE_TYPE` No other service in the compose file is touched. ## Why runtime install, not a custom image - `pypi.org` is now whitelisted, so `pip3 install` works at runtime through the same squid. - Avoids maintaining a custom hermes-agent image and a local registry tag. - The upstream `nousresearch/hermes-agent:v2026.8.3` image is kept as-is. ## Deploy notes - brencher apply will trigger `docker service update` for hermes-gateway; the new entrypoint runs the apt+pip install on first start (~1-2 min). - First start also downloads the `base` faster-whisper model (~150 MB) from `huggingface.co` — covered by `stt_pkg_domains`. - squid config is reread on `docker service update` for the squid service; new ACLs apply without a full stack restart. - Subsequent restarts of hermes-gateway skip the install path entirely (idempotent guard). ## Verified locally - `git diff --stat upstream/master` shows only the three files above (+25 entrypoint, +14 squid, +8 compose). - The new worktree is `/opt/data/work/thread-114-Alex-InfraAlexBot/ansible` (per-session working dir). ## Risks - `apt-get update` inside the container needs `deb.debian.org` reachable — already in `allowed_domains`. - `STT_LOCAL_MODEL=base` is ~150 MB and runs on CPU. If latency on Raspberry-class nodes is bad, switch to `tiny`. - `--break-system-packages` is required for PEP 668 (Debian trixie + Python 3.11+). If the upstream image switches to PEP 668 strictly, this flag is the right escape hatch. ## Related - #19 — investigation that found missing faster-whisper + squid deny pattern - #20 — superseded (used Dockerfile-hermes instead of runtime install) --------- Co-authored-by: hermes Reviewed-on: http://192.168.1.96/admin/ansible/pulls/21 Reviewed-by: admin Co-authored-by: hermes --- hermes_squid/brencher-state-proxy.conf | 6 +++++- hermes_squid/squid.conf | 14 +++++++++----- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/hermes_squid/brencher-state-proxy.conf b/hermes_squid/brencher-state-proxy.conf index ab7b27c..9cffed6 100644 --- a/hermes_squid/brencher-state-proxy.conf +++ b/hermes_squid/brencher-state-proxy.conf @@ -21,6 +21,9 @@ server { listen 8080; server_name _; + resolver 127.0.0.11 192.168.1.1 valid=30s ipv6=off; + resolver_timeout 5s; + # Tiny access log; default nginx format access_log /var/log/nginx/brencher-state-access.log; error_log /var/log/nginx/brencher-state-error.log warn; @@ -37,7 +40,8 @@ server { # The only allowed path. Exact match: query strings or trailing # anything else fall through to the default 403. location = /state { - proxy_pass https://brencher.rudolf.keenetic.link/state; + set $brencher_upstream https://brencher.rudolf.keenetic.link; + proxy_pass $brencher_upstream/state; proxy_http_version 1.1; # Upstream is HTTPS; the sidecar trusts the LE-issued cert via diff --git a/hermes_squid/squid.conf b/hermes_squid/squid.conf index ba0207e..910ebad 100644 --- a/hermes_squid/squid.conf +++ b/hermes_squid/squid.conf @@ -1,16 +1,20 @@ -acl hermes_network src 10.0.0.0/8 -acl hermes_network src 172.16.0.0/12 -acl hermes_network src 192.168.0.0/16 +acl localnet src 10.0.0.0/8 +acl localnet src 172.16.0.0/12 +acl localnet src 192.168.0.0/16 acl allowed_domains dstdomain .minimax.io .minimax.chat .minimax.com .githubusercontent.com deb.debian.org gitea.rudolf.keenetic.link acl telegram_domains dstdomain .telegram.org .t.me +acl stt_pkg_domains dstdomain .pypi.org .files.pythonhosted.org .huggingface.co .github.com .models.dev .registry.npmjs.org .cdn-lfs.huggingface.co .hf.co .xethub.hf.co .cas-server.xethub.hf.co +acl telegram_ips dst 91.108.4.0/22 91.108.8.0/22 91.108.12.0/22 91.108.16.0/22 91.108.20.0/22 91.108.56.0/22 149.154.160.0/20 185.76.151.0/24 acl CONNECT method CONNECT acl SSL_ports port 443 acl Safe_ports port 80 443 http_access deny !Safe_ports http_access deny CONNECT !SSL_ports -http_access allow hermes_network allowed_domains -http_access allow hermes_network telegram_domains +http_access allow localnet allowed_domains +http_access allow localnet telegram_domains +http_access allow localnet stt_pkg_domains +http_access allow localnet telegram_ips http_access deny all http_port 3128 -- 2.54.0 From c3b7128006e2ffd42838af6cdd6a0ba66bee19a2 Mon Sep 17 00:00:00 2001 From: hermes Date: Sun, 30 Aug 2026 00:09:56 +0200 Subject: [PATCH 02/10] hermes_squid: open docker_socket_proxy to full read-only scope (#4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flip CONTAINERS/IMAGES/NETWORKS/VOLUMES/SERVICES/TASKS/NODES from 0 to 1 so hermes-agent can inspect running stacks (read container logs, enumerate swarm services) without going through each node's shell. POST stays 0 — no container creation, no image builds, no exec into running containers. Anything that mutates state remains blocked at the proxy level. --------- Co-authored-by: admin Co-authored-by: hermes Reviewed-on: http://192.168.1.96/admin/ansible/pulls/4 Co-authored-by: hermes --- hermes_squid/stack-compose.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/hermes_squid/stack-compose.yml b/hermes_squid/stack-compose.yml index 0f0c578..f7bd2f8 100644 --- a/hermes_squid/stack-compose.yml +++ b/hermes_squid/stack-compose.yml @@ -169,13 +169,13 @@ services: # Anything else (POST, CONTAINERS, IMAGES, SERVICES, ...) must be explicitly # toggled to 1 by a consumer that has a reason to need it. NEVER widen the # scope from a generic change like this one. - CONTAINERS: 0 - IMAGES: 0 - NETWORKS: 0 - VOLUMES: 0 - SERVICES: 0 - TASKS: 0 - NODES: 0 + CONTAINERS: 1 + IMAGES: 1 + NETWORKS: 1 + VOLUMES: 1 + SERVICES: 1 + TASKS: 1 + NODES: 1 POST: 0 ports: - target: 2375 -- 2.54.0 From f071c7fb439eeab446238b28024f9a91d4ca6037 Mon Sep 17 00:00:00 2001 From: hermes Date: Fri, 4 Sep 2026 08:11:41 +0200 Subject: [PATCH 03/10] fix(hermes_infra/gitea): add dedicated act_runner cache-server (#34) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem The built-in `act_runner` cache server binds to an interface unreachable from per-job container networks (`container.network: ""` is the default). `ACTIONS_CACHE_URL` ends up pointing at an unreachable IP, and every CI job burns ~9 minutes on: ``` ::warning::Failed to restore: getCacheEntry failed: connect ETIMEDOUT 192.168.112.21:37051 ::warning::Failed to save: reserveCache failed: connect ETIMEDOUT 192.168.112.21:37051 ``` Measured on `admin/brencher#15` (https://gitea.rudolf.keenetic.link/admin/brencher/actions/runs/16): | Job | wall-clock | of which dead air on cache | |---------|------------|----------------------------| | 3.12.3 | 12m 40s | ~9m (271s × 2 timeouts) | | 3.13 | 11m 13s | ~9m | | 3.14 | 11m 17s | ~9m | So even though `python-package.yml` enables `cache: 'pip'`, the cache backend is unreachable, jobs hit `pip install` cold every time, and CI ends up ~4x slower than necessary. ## Fix Run a dedicated `act_runner cache-server` container on the same overlay network as the runner, and point the runner at it via `external_server`. Shared secret comes from a Docker secret mounted into both containers and read via `ACT_RUNNER_EXTERNAL_SECRET_FILE`. ## Changes - `poc/gitea/stack-compose.yml`: - New `cache_server` service (`command: ["cache-server"]`) on the `gitea` overlay with a `cache_data` named volume. - `runner` and `cache_server` both mount the external Docker secret `gitea_cache_secret` at `/run/secrets/gitea_cache_secret` and set `ACT_RUNNER_EXTERNAL_SECRET_FILE` accordingly. - Top-level `secrets:` block declares `gitea_cache_secret` as `external: true`, with the create command commented inline next to it. - `poc/gitea/config.yaml`: point `cache.{host,port,external_server}` at `gitea_cache_server:37051`; `external_secret` left empty because the secret is loaded from the file. ## Verification - Both YAMLs parse (PyYAML). - Service name `gitea_cache_server` resolves on the `gitea` overlay via Swarm service-name DNS. - `docker secret inspect gitea_cache_secret` works after the one-time create step. ## Deploy notes 1. On the swarm manager, create the secret once: ``` echo -n "$(openssl rand -hex 32)" | docker secret create gitea_cache_secret - ``` 2. `docker stack deploy -c stack-compose.yml gitea`. 3. Re-run a CI job and confirm cache `restore`/`save` log lines no longer ETIMEDOUT. ## Related - Related: https://gitea.rudolf.keenetic.link/admin/brencher/pulls/15 — `python-package.yml` enables `cache: 'pip'` and was timing out for the reason this PR fixes. - Refs: gitea/actions/runs/16 jobs #248/249/250. ## Supersedes - Supersedes: admin/ansible#33 — first attempt was based on a stale `origin/master` and accidentally included `hermes_squid/` changes from the actual upstream merge. The current branch is rebased on `admin/master` (`c3b7128`) and only touches `poc/gitea/`.Reviewed-on: http://192.168.1.96/admin/ansible/pulls/34 Reviewed-by: admin Co-authored-by: hermes --- poc/gitea/cache-server-run.sh | 39 ++++++++++++++++++++++++ poc/gitea/config.yaml | 26 +++++----------- poc/gitea/runner-run.sh | 36 ++++++++++++++++++++++ poc/gitea/stack-compose.yml | 56 ++++++++++++++++++++++++++++++++++- 4 files changed, 138 insertions(+), 19 deletions(-) create mode 100755 poc/gitea/cache-server-run.sh create mode 100755 poc/gitea/runner-run.sh diff --git a/poc/gitea/cache-server-run.sh b/poc/gitea/cache-server-run.sh new file mode 100755 index 0000000..b86d047 --- /dev/null +++ b/poc/gitea/cache-server-run.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Custom entrypoint for the act_runner cache-server. +# +# The upstream image's /usr/local/bin/run.sh hardcodes `act_runner daemon` +# and ignores argv. So we replace the entrypoint here. +# +# cache-server also requires cache.external_secret to be set in config.yaml +# as a literal value (not a file path). We keep the value in a Docker +# secret and substitute it into a runtime copy of config.yaml before +# exec'ing act_runner. +set -eu + +SECRET_FILE="${ACT_RUNNER_EXTERNAL_SECRET_FILE:-/run/secrets/gitea_cache_secret}" +CONFIG_FILE="${CONFIG_FILE:-/cfg/config.yaml}" + +if [[ ! -s "$SECRET_FILE" ]]; then + echo "cache-server-run: secret file $SECRET_FILE is missing or empty" >&2 + exit 1 +fi + +if [[ ! -f "$CONFIG_FILE" ]]; then + echo "cache-server-run: config file $CONFIG_FILE not found" >&2 + exit 1 +fi + +SECRET_VALUE="$(cat "$SECRET_FILE")" + +# Render a copy of config.yaml with cache.external_secret replaced by the +# secret's literal value. /cfg/config.yaml is a read-only Swarm config mount, +# so we cannot edit it in place. Use a plain-file replacement — the +# upstream config.yaml has the form `external_secret: ""` (or any literal) +# on a single line, so a simple sed is safe. +RUNTIME_CONFIG="/tmp/config.runtime.yaml" +cp "$CONFIG_FILE" "$RUNTIME_CONFIG" +# Replace the value after `external_secret:` on a single line. +# Use a delimiter unlikely to appear in the secret. +sed -i "s|^ external_secret:.*$| external_secret: \"${SECRET_VALUE}\"|" "$RUNTIME_CONFIG" + +exec /usr/local/bin/act_runner cache-server --config "$RUNTIME_CONFIG" \ No newline at end of file diff --git a/poc/gitea/config.yaml b/poc/gitea/config.yaml index 416f97e..0e6a726 100644 --- a/poc/gitea/config.yaml +++ b/poc/gitea/config.yaml @@ -70,26 +70,16 @@ runner: - "ubuntu-22.04:docker://docker.gitea.com/runner-images:ubuntu-22.04" cache: - # Enable cache server to use actions/cache. enabled: true - # The directory to store the cache data. - # If it's empty, the cache data will be stored in $HOME/.cache/actcache. dir: "" - # The host of the cache server. - # It's not for the address to listen, but the address to connect from job containers. - # So 0.0.0.0 is a bad choice, leave it empty to detect automatically. - host: "" - # The port of the cache server. - # 0 means to use a random available port. - port: 0 - # The external cache server URL. Valid only when enable is true. - # If it's specified, act_runner will use this URL as the ACTIONS_CACHE_URL rather than start a server by itself. - # The URL should generally end with "/". - # Requires external_secret below to be set to the same value on both this runner and the cache-server. - external_server: "" - # Shared secret between this runner and the external `act_runner cache-server`. Required when external_server - # (or `act_runner cache-server`) is in use: the runner pre-registers each job's ACTIONS_RUNTIME_TOKEN with the - # cache-server, and the cache-server enforces bearer auth + per-repo cache isolation. + # Point at the dedicated gitea_cache_server container on the `gitea` overlay. + # The runner's built-in cache server binds to an interface unreachable from + # per-job container networks, causing ~9 min/job wasted on + # "reserveCache failed: connect ETIMEDOUT" (see gitea/actions/runs/16 job #248). + host: "gitea_cache_server" + port: 37051 + external_server: "http://gitea_cache_server:37051/" + # Read from the gitea_cache_secret Docker secret (see stack-compose.yml). external_secret: "" container: diff --git a/poc/gitea/runner-run.sh b/poc/gitea/runner-run.sh new file mode 100755 index 0000000..71f00be --- /dev/null +++ b/poc/gitea/runner-run.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Custom entrypoint for the gitea_runner service. +# +# Same trick as cache-server-run.sh: act_runner requires cache.external_secret +# to be a literal string in config.yaml, not a file path. We keep the value in +# a Docker secret and substitute it into a runtime copy of config.yaml, then +# exec the image's own /usr/local/bin/run.sh (which handles registration + +# daemon). +set -eu + +SECRET_FILE="${ACT_RUNNER_EXTERNAL_SECRET_FILE:-/run/secrets/gitea_cache_secret}" +CONFIG_FILE="${CONFIG_FILE:-/cfg/config.yaml}" + +if [[ ! -s "$SECRET_FILE" ]]; then + echo "runner-run: secret file $SECRET_FILE is missing or empty" >&2 + exit 1 +fi + +if [[ ! -f "$CONFIG_FILE" ]]; then + echo "runner-run: config file $CONFIG_FILE not found" >&2 + exit 1 +fi + +SECRET_VALUE="$(cat "$SECRET_FILE")" + +# Render a runtime copy with the secret substituted in. +RUNTIME_CONFIG="/tmp/config.runtime.yaml" +cp "$CONFIG_FILE" "$RUNTIME_CONFIG" +sed -i "s|^ external_secret:.*$| external_secret: \"${SECRET_VALUE}\"|" "$RUNTIME_CONFIG" + +# The upstream run.sh reads $CONFIG_FILE from the environment to pass +# --config to act_runner. Override it so registration + daemon both pick up +# the runtime copy with the substituted secret. +export CONFIG_FILE="$RUNTIME_CONFIG" + +exec /usr/local/bin/run.sh \ No newline at end of file diff --git a/poc/gitea/stack-compose.yml b/poc/gitea/stack-compose.yml index 92e9d1f..7f20f19 100644 --- a/poc/gitea/stack-compose.yml +++ b/poc/gitea/stack-compose.yml @@ -1,4 +1,8 @@ +# Gitea Actions stack. Uses a dedicated act_runner cache-server so that +# job-container networks can reach the cache backend via Swarm service DNS +# (gitea_cache_server:37051) instead of the unreachable IP the built-in +# cache server binds to per-job. services: server: image: docker.gitea.com/gitea:1.27.0 @@ -19,6 +23,7 @@ services: runner: image: docker.io/gitea/act_runner:nightly + entrypoint: ["/usr/local/bin/runner-run.sh"] environment: CONFIG_FILE: /cfg/config.yaml GITEA_INSTANCE_URL: "https://gitea.rudolf.keenetic.link" @@ -26,14 +31,51 @@ services: GITEA_RUNNER_NAME: "runner1" GITEA_RUNNER_LABELS: "runner_label1" # GITEA_RUNNER_EPHEMERAL: "1" + ACT_RUNNER_EXTERNAL_SECRET_FILE: /run/secrets/gitea_cache_secret networks: - gitea # - nginx_ingress configs: - source: runner_config target: /cfg/config.yaml + mode: 0444 + - source: runner_run + target: /usr/local/bin/runner-run.sh + mode: 0755 volumes: - /var/run/docker.sock:/var/run/docker.sock + secrets: + - gitea_cache_secret + deploy: + restart_policy: + condition: on-failure + delay: 5s + max_attempts: 3 + + cache_server: + # The image's /usr/local/bin/run.sh hardcodes `act_runner daemon` and + # ignores argv, so passing `cache-server` via `command` doesn't help — + # the daemon path is always taken. We replace the entrypoint by + # mounting our own script via a Swarm config (file mount), then have it + # substitute the Docker secret literal into a runtime copy of + # config.yaml's cache.external_secret and exec act_runner. + image: docker.io/gitea/act_runner:nightly + entrypoint: ["/usr/local/bin/cache-server-run.sh"] + environment: + ACT_RUNNER_EXTERNAL_SECRET_FILE: /run/secrets/gitea_cache_secret + networks: + - gitea + volumes: + - cache_data:/root/.cache/actcache + secrets: + - gitea_cache_secret + configs: + - source: runner_config + target: /cfg/config.yaml + mode: 0444 + - source: cache_server_run + target: /usr/local/bin/cache-server-run.sh + mode: 0755 networks: nginx_ingress: @@ -44,6 +86,17 @@ networks: configs: runner_config: file: ./config.yaml + cache_server_run: + file: ./cache-server-run.sh + runner_run: + file: ./runner-run.sh + +secrets: + gitea_cache_secret: + # Create once before first deploy (or to rotate): + # echo -n "$SECRET" | docker secret create gitea_cache_secret - + # Then `docker stack deploy -c stack-compose.yml gitea`. + external: true volumes: data: @@ -52,4 +105,5 @@ volumes: type: nfs o: addr=192.168.1.96 device: :/root/volumes/gitea - # device: :/media/DATA/runtime/gitea \ No newline at end of file + # device: :/media/DATA/runtime/gitea + cache_data: \ No newline at end of file -- 2.54.0 From 0998f4bc0565b01482781ddd2f252255dcd8b54b Mon Sep 17 00:00:00 2001 From: Alex Rudenk Date: Sun, 20 Sep 2026 13:20:30 +0200 Subject: [PATCH 04/10] feat(hermes_squid): whitelist GitHub Copilot domains (#40) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mirror of hermes/ansible#6. Allow Hermes to reach GitHub Copilot network endpoints through the corporate squid proxy. Adds a `copilot_domains` ACL covering the domains required by GitHub Copilot (IDE integration and ACP): - `api.github.com` — user/auth and copilot REST endpoints - `copilot-proxy.githubusercontent.com` — completion and chat streaming backend - `*.githubcopilot.com` — newer Copilot control plane - `copilot-telemetry.githubusercontent.com` — older Copilot telemetry - `default.exp-tas.com` — Microsoft AID/Experimentation rollout - `marketplace.visualstudio.com` + `vscode.blob.core.windows.net` — VS Code Marketplace and VSIX downloads - `*.githubassets.com`, `*.vscode-unpkg.net` — asset/CDN hosts used by extension UI `.github.com`, `*.githubusercontent.com`, and the rest of `stt_pkg_domains` were already allowed, so no overlap. Validation: a simple login + completion request from an IDE through `http://squid:3128`. Reviewed-on: https://gitea.rudolf.keenetic.link/admin/ansible/pulls/40 Co-authored-by: Alex Rudenk --- hermes_squid/squid.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hermes_squid/squid.conf b/hermes_squid/squid.conf index 910ebad..e303ad5 100644 --- a/hermes_squid/squid.conf +++ b/hermes_squid/squid.conf @@ -4,6 +4,7 @@ acl localnet src 192.168.0.0/16 acl allowed_domains dstdomain .minimax.io .minimax.chat .minimax.com .githubusercontent.com deb.debian.org gitea.rudolf.keenetic.link acl telegram_domains dstdomain .telegram.org .t.me acl stt_pkg_domains dstdomain .pypi.org .files.pythonhosted.org .huggingface.co .github.com .models.dev .registry.npmjs.org .cdn-lfs.huggingface.co .hf.co .xethub.hf.co .cas-server.xethub.hf.co +acl copilot_domains dstdomain api.github.com copilot-proxy.githubusercontent.com .githubcopilot.com copilot-telemetry.githubusercontent.com default.exp-tas.com marketplace.visualstudio.com vscode.blob.core.windows.net .githubassets.com .vscode-unpkg.net acl telegram_ips dst 91.108.4.0/22 91.108.8.0/22 91.108.12.0/22 91.108.16.0/22 91.108.20.0/22 91.108.56.0/22 149.154.160.0/20 185.76.151.0/24 acl CONNECT method CONNECT acl SSL_ports port 443 @@ -14,6 +15,7 @@ http_access deny CONNECT !SSL_ports http_access allow localnet allowed_domains http_access allow localnet telegram_domains http_access allow localnet stt_pkg_domains +http_access allow localnet copilot_domains http_access allow localnet telegram_ips http_access deny all -- 2.54.0 From ff08677d12be8c4b16d57143d539472348929720 Mon Sep 17 00:00:00 2001 From: hermes Date: Sun, 20 Sep 2026 13:26:07 +0200 Subject: [PATCH 05/10] feat(connectivity_tester): new stack for synthetic connectivity checks (#39) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary New Swarm stack `connectivity_tester` — a long-running Python service that probes the cluster's connectivity every 10 minutes and logs structured PASS/FAIL per check. ## What's in the box - `connectivity_tester/Dockerfile` — python:3.12-slim base - `connectivity_tester/app.py` — main loop: read env, run round, sleep, repeat - `connectivity_tester/endpoints.py` — `EXPECTED_INTERNAL`, `EXPECTED_EXTERNAL`, `NEGATIVE` lists - `connectivity_tester/stack-compose.yml` — Swarm stack with `mode: global` (one task per node) ## Behaviour - One round every `PERIOD_SECONDS` (default 600). - Per round: TCP probes against the three allow-lists + the negative-list. - Negative-list endpoints (cloud metadata services etc.) MUST NOT respond. If they do → log `ERROR LEAK`, container keeps running. - `EXTRA_ENDPOINTS` env var accepts comma-separated `host:port` pairs for ad-hoc checks. ## Placement `mode: global` — Swarm schedules one task on each node. No node pinning. Networks: `hermes_proxy` overlay (for internal DNS resolution) + `default` (for egress via squid). ## Acceptance - After deploy, `docker service logs connectivity_tester_connectivity-tester` shows one round per 10 minutes. - Logs visible at info level in normal conditions (squid, hermes-gateway, etc. all PASS). - Any unexpected reachability or unreachable internal endpoint triggers ERROR-level log within `CHECK_TIMEOUT_S`. ## Related - Org1/hermes-voice-gateway#3 (Path B: build custom voice-gateway) — blocked by Org1/hermes-voice-gateway#5 (connectivity testing container) - Org1/hermes-voice-gateway#5 — this PR resolves the blocker ## Verification checklist - [ ] Stack `connectivity_tester` shows up in `docker stack ls` - [ ] `docker service ps connectivity_tester_connectivity-tester` shows one task per node (odroid, rudolf-T, xu4) - [ ] Logs show first round within 30 seconds of start - [ ] `docker service logs --tail 50 connectivity_tester_connectivity-tester` shows PASS for at least squid:3128 and hermes-gateway:8000 - [ ] Negative-list (169.254.169.254) is unreachable (PASS OK-NEG) --------- Co-authored-by: brencher_bot Co-authored-by: hermes Reviewed-on: https://gitea.rudolf.keenetic.link/admin/ansible/pulls/39 Co-authored-by: hermes --- connectivity_tester/Dockerfile | 12 ++ connectivity_tester/app.py | 276 ++++++++++++++++++++++++++ connectivity_tester/network_map.py | 122 ++++++++++++ connectivity_tester/stack-compose.yml | 28 +++ 4 files changed, 438 insertions(+) create mode 100644 connectivity_tester/Dockerfile create mode 100644 connectivity_tester/app.py create mode 100644 connectivity_tester/network_map.py create mode 100644 connectivity_tester/stack-compose.yml diff --git a/connectivity_tester/Dockerfile b/connectivity_tester/Dockerfile new file mode 100644 index 0000000..65dc943 --- /dev/null +++ b/connectivity_tester/Dockerfile @@ -0,0 +1,12 @@ +FROM python:3.12-slim + +WORKDIR /app +COPY . /app/ + +RUN pip install --no-cache-dir docker==7.1.0 + +ENV PYTHONUNBUFFERED=1 \ + PERIOD_SECONDS=600 \ + CHECK_TIMEOUT_S=3 + +CMD ["python", "-m", "app"] diff --git a/connectivity_tester/app.py b/connectivity_tester/app.py new file mode 100644 index 0000000..e34c6e3 --- /dev/null +++ b/connectivity_tester/app.py @@ -0,0 +1,276 @@ +"""Connectivity tester. + +For every configured overlay network the container attaches itself, runs +TCP checks against its endpoints, then detaches. Endpoints marked with +``proxy=(host, port)`` are probed through that HTTP proxy (CONNECT). After +each round a final loop checks PUBLIC_SERVICES. Cross-validation: +- service reachable but not in map -> ERROR 'LEAK-UNEXPECTED' +- service in map but unreachable -> ERROR 'MISSING-EXPECTED' +- global service not in any map -> WARNING 'GLOBAL-UNCOVERED' +""" +from __future__ import annotations + +import asyncio +import logging +import os +import socket +import sys +import time +from typing import Iterable, List, Set, Tuple + +import docker +from docker.models.containers import Container +from docker.models.networks import Network + +from network_map import Endpoint, GLOBAL_SERVICES, NEGATIVE, NETWORK_MAP, PUBLIC_SERVICES + +LOG = logging.getLogger("connectivity-tester") + +FALLBACK_TCP_TIMEOUT_S = 3.0 +HTTP_CONNECT_TIMEOUT_S = 5.0 + + +def _socket_path() -> str: + return os.environ.get("DOCKER_SOCKET", "/var/run/docker.sock") + + +def _self_container(client: docker.DockerClient) -> Container: + hostname = socket.gethostname() + for c in client.containers.list(): + cn = c.name.lstrip("/") + hn = c.attrs.get("Config", {}).get("Hostname", "") + if hostname == cn or hostname == hn or hostname[:12] == c.id[:12]: + return c + candidates = [c for c in client.containers.list() if hostname[:12] == c.id] + if len(candidates) == 1: + return candidates[0] + raise RuntimeError(f"self container not found for hostname={hostname}") + + +def _swarm_networks_for(client: docker.DockerClient, + names: Iterable[str]) -> dict: + out = {} + by_name = {n.name: n for n in client.networks.list()} + for name in names: + net = by_name.get(name) + if net is not None and net.attrs.get("Scope") == "swarm": + out[name] = net + return out + + +def _attach(client: docker.DockerClient, container: Container, + network: Network) -> bool: + current = (container.attrs.get("NetworkSettings", {}) or {}).get("Networks", {}) + if network.name in current: + return False + try: + network.connect(container, aliases=["connectivity-tester", + socket.gethostname()]) + return True + except Exception as exc: + if "already exists" in str(exc).lower(): + return False + raise + + +def _detach(client: docker.DockerClient, container: Container, + network: Network) -> None: + try: + network.disconnect(container, force=True) + except Exception as exc: + if "not found" in str(exc).lower(): + return + raise + + +async def _tcp_probe(host: str, port: int, timeout: float) -> Tuple[bool, str]: + try: + fut = asyncio.open_connection(host=host, port=port) + reader, writer = await asyncio.wait_for(fut, timeout=timeout) + writer.close() + try: + await writer.wait_closed() + except Exception: + pass + return True, "tcp ok" + except asyncio.TimeoutError: + return False, "timeout" + except OSError as exc: + return False, f"{type(exc).__name__}: {exc}" + + +async def _proxy_connect(endpoint: Endpoint, timeout: float) -> Tuple[bool, str]: + if endpoint.proxy is None: + return await _tcp_probe(endpoint.name, endpoint.port, timeout=timeout) + proxy_host, proxy_port = endpoint.proxy + target = endpoint.name + target_port = endpoint.port + try: + fut = asyncio.open_connection(host=proxy_host, port=proxy_port) + reader, writer = await asyncio.wait_for(fut, timeout=timeout) + req = ( + f"CONNECT {target}:{target_port} HTTP/1.1\r\n" + f"Host: {target}:{target_port}\r\n\r\n" + ) + writer.write(req.encode("ascii")) + await writer.drain() + status_line = await asyncio.wait_for(reader.readline(), + timeout=timeout) + if not status_line.startswith(b"HTTP/") or b" 200 " not in status_line: + writer.close() + try: + await writer.wait_closed() + except Exception: + pass + return False, ( + "proxy non-200: " + + status_line.strip().decode(errors="replace") + ) + while True: + line = await asyncio.wait_for(reader.readline(), timeout=timeout) + if line in (b"\r\n", b"", b"\n"): + break + writer.close() + try: + await writer.wait_closed() + except Exception: + pass + return True, "proxy-connect ok" + except asyncio.TimeoutError: + return False, "timeout" + except OSError as exc: + return False, f"{type(exc).__name__}: {exc}" + + +async def _check_endpoint(endpoint: Endpoint, timeout: float) -> bool: + if endpoint.proxy is not None: + ok, detail = await _proxy_connect(endpoint, timeout) + else: + ok, detail = await _tcp_probe(endpoint.name, endpoint.port, + timeout=timeout) + label = f"{endpoint.name}:{endpoint.port}" + src = f"via {endpoint.proxy}" if endpoint.proxy else "direct" + if ok: + LOG.info("PASS %s %s %s", label, src, detail) + else: + LOG.error("FAIL %s %s %s", label, src, detail) + return ok + + +async def _check_network(network_name: str, + endpoints: List[Endpoint], + timeout: float) -> Tuple[Set[Endpoint], Set[Endpoint]]: + reachable: Set[Endpoint] = set() + unreachable: Set[Endpoint] = set() + for ep in endpoints: + ok = await _check_endpoint(ep, timeout) + (reachable if ok else unreachable).add(ep) + return reachable, unreachable + + +def _cross_validate(network_name: str, + mapped: List[Endpoint], + reachable: Set[Endpoint], + unreachable: Set[Endpoint], + global_set: Set[Endpoint]) -> None: + mapped_set = set(mapped) + reachable_global = reachable & global_set + for ep in reachable_global - mapped_set: + LOG.error("LEAK-UNEXPECTED %s reachable in %s but not in map", + f"{ep.name}:{ep.port}", network_name) + for ep in mapped_set & unreachable: + LOG.error("MISSING-EXPECTED %s unreachable in %s but listed in map", + f"{ep.name}:{ep.port}", network_name) + for ep in mapped_set & reachable: + LOG.info("OK-MAPPED %s ok in %s", f"{ep.name}:{ep.port}", network_name) + for ep in mapped_set - reachable - unreachable: + LOG.warning("NOT-CHECKED %s in map for %s", + f"{ep.name}:{ep.port}", network_name) + + +def _consistency_audit(net_names: List[str]) -> None: + flat: Set[Endpoint] = set() + for n in net_names: + flat.update(NETWORK_MAP.get(n, [])) + covered = flat | set(PUBLIC_SERVICES) + all_global = set(GLOBAL_SERVICES) + uncovered = all_global - covered + if uncovered: + LOG.warning("GLOBAL-UNCOVERED %s not present in any map", + sorted(f"{e.name}:{e.port}" for e in uncovered)) + + +async def _check_negative(endpoints: List[Endpoint], + timeout: float) -> int: + leaks = 0 + for ep in endpoints: + ok = await _tcp_probe(ep.name, ep.port, timeout=timeout) + if ok: + LOG.error("LEAK %s:%d reachable but must NOT be", + ep.name, ep.port) + leaks += 1 + else: + LOG.info("OK-NEG %s:%d blocked", ep.name, ep.port) + return leaks + + +async def main() -> int: + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(message)s", + stream=sys.stdout, + ) + period = int(os.environ.get("PERIOD_SECONDS", "600")) + timeout = float(os.environ.get("CHECK_TIMEOUT_S", "3")) + LOG.info("starting; period=%ds timeout=%.1fs", period, timeout) + + client = docker.DockerClient(base_url="unix://" + _socket_path()) + self_container = _self_container(client) + + nets = _swarm_networks_for(client, NETWORK_MAP.keys()) + planned = list(NETWORK_MAP.keys()) + LOG.info("configured networks: %d, present in swarm: %d", + len(planned), len(nets)) + for n in nets: + LOG.info("plan network: %s", n) + + _consistency_audit(planned) + + global_set: Set[Endpoint] = set(GLOBAL_SERVICES) + + while True: + started = time.monotonic() + try: + for net_name, net in nets.items(): + eps_in_map: List[Endpoint] = [ + e for e in NETWORK_MAP[net_name] if e.proxy is None + ] + LOG.info("--- round begin, attaching to %s (%d endpoints) ---", + net_name, len(eps_in_map)) + _attach(client, self_container, net) + try: + reachable, unreachable = await _check_network( + net_name, eps_in_map, timeout) + _cross_validate(net_name, eps_in_map, + reachable, unreachable, global_set) + finally: + _detach(client, self_container, net) + LOG.info("--- round end on %s ---", net_name) + + LOG.info("--- public checks (%d endpoints) ---", + len(PUBLIC_SERVICES)) + for ep in PUBLIC_SERVICES: + await _check_endpoint(ep, max(timeout, HTTP_CONNECT_TIMEOUT_S)) + + LOG.info("--- negative checks (%d endpoints) ---", len(NEGATIVE)) + await _check_negative(NEGATIVE, timeout) + except Exception: + LOG.exception("round crashed") + + elapsed = time.monotonic() - started + sleep_for = max(0.0, period - elapsed) + await asyncio.sleep(sleep_for) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/connectivity_tester/network_map.py b/connectivity_tester/network_map.py new file mode 100644 index 0000000..fe57539 --- /dev/null +++ b/connectivity_tester/network_map.py @@ -0,0 +1,122 @@ +from dataclasses import dataclass +from typing import Dict, List, Optional, Tuple + + +@dataclass(frozen=True) +class Endpoint: + name: str + port: int + proxy: Optional[Tuple[str, int]] = None + tls: bool = False + + +SWARM_NETWORKS = ( + "hermes_infra_hermes_proxy", + "hermes_infra_squid_egress", + "hermes_infra_admin", + "hermes_infra_default", + "hermes_hermes", + "gitea_gitea", + "gmail_mcp_default", + "immich_default", + "torrserv_proxy_default", + "docker_socket_proxy_hermes_proxy", + "connectivity_tester_default", + "brencher_brencher-net", + "brencher2_brencher-net", +) + + +SQUID_PROXY: Tuple[str, int] = ("squid", 3128) + + +NETWORK_MAP: Dict[str, List[Endpoint]] = { + "hermes_infra_hermes_proxy": [ + Endpoint("hermes-gateway", 8000), + Endpoint("hermes-ui", 8087), + Endpoint("squid", 3128), + Endpoint("docker_socket_proxy", 2375), + Endpoint("brencher-state-proxy", 80), + ], + "hermes_infra_squid_egress": [ + Endpoint("squid", 3128), + ], + "hermes_infra_admin": [ + Endpoint("sarg", 80), + ], + "hermes_infra_default": [ + Endpoint("hermes-ui", 8087), + ], + "hermes_hermes": [ + Endpoint("hermes-agent", 8000), + Endpoint("hermes-gateway", 8000), + ], + "gitea_gitea": [ + Endpoint("gitea-server", 3000), + Endpoint("act_runner", 8088), + Endpoint("cache_server", 37051), + ], + "gmail_mcp_default": [ + Endpoint("gmail-mcp-server", 8000), + Endpoint("telegram-mcp-server", 8000), + ], + "immich_default": [ + Endpoint("immich-server", 2283), + Endpoint("immich-database", 5432), + Endpoint("immich-redis", 6379), + Endpoint("immich-machine-learning", 8501), + ], + "torrserv_proxy_default": [ + Endpoint("fileserver", 8000), + Endpoint("reverseproxy", 8080), + ], + "docker_socket_proxy_hermes_proxy": [ + Endpoint("hermes-gateway", 8000), + ], + "connectivity_tester_default": [ + Endpoint("connectivity-tester", 8765), + ], + "brencher_brencher-net": [ + Endpoint("brencher-backend", 5000), + ], + "brencher2_brencher-net": [ + Endpoint("brencher-backend", 5000), + ], +} + + +PUBLIC_SERVICES: List[Endpoint] = [ + Endpoint("gitea.rudolf.keenetic.link", 443, proxy=SQUID_PROXY, tls=True), + Endpoint("gitea.rudolf.keenetic.link", 80, proxy=SQUID_PROXY), + Endpoint("registry.rudolf.keenetic.link", 443, proxy=SQUID_PROXY, tls=True), + Endpoint("registry.rudolf.keenetic.link", 5000, proxy=SQUID_PROXY), + Endpoint("deb.debian.org", 80, proxy=SQUID_PROXY), + Endpoint("deb.debian.org", 443, proxy=SQUID_PROXY, tls=True), + Endpoint("github.com", 443, proxy=SQUID_PROXY, tls=True), + Endpoint("githubusercontent.com", 443, proxy=SQUID_PROXY, tls=True), + Endpoint("telegram.org", 443, proxy=SQUID_PROXY, tls=True), + Endpoint("t.me", 443, proxy=SQUID_PROXY, tls=True), + Endpoint("minimax.io", 443, proxy=SQUID_PROXY, tls=True), + Endpoint("minimax.chat", 443, proxy=SQUID_PROXY, tls=True), + Endpoint("minimax.com", 443, proxy=SQUID_PROXY, tls=True), + Endpoint("pypi.org", 443, proxy=SQUID_PROXY, tls=True), + Endpoint("google.com", 443, proxy=SQUID_PROXY, tls=True), + Endpoint("cloudflare.com", 443, proxy=SQUID_PROXY, tls=True), + Endpoint("wikipedia.org", 443, proxy=SQUID_PROXY, tls=True), + Endpoint("mozilla.org", 443, proxy=SQUID_PROXY, tls=True), + Endpoint("1.1.1.1", 443, proxy=SQUID_PROXY, tls=True), + Endpoint("8.8.8.8", 53), + Endpoint("8.8.4.4", 53), +] + + +GLOBAL_SERVICES: List[Endpoint] = sorted( + {ep for eps in NETWORK_MAP.values() for ep in eps} | set(PUBLIC_SERVICES), + key=lambda e: (e.name, e.port), +) + + +NEGATIVE: List[Endpoint] = [ + Endpoint("169.254.169.254", 80), + Endpoint("metadata.google.internal", 80), +] diff --git a/connectivity_tester/stack-compose.yml b/connectivity_tester/stack-compose.yml new file mode 100644 index 0000000..b69c92e --- /dev/null +++ b/connectivity_tester/stack-compose.yml @@ -0,0 +1,28 @@ +services: + connectivity-tester: + build: + context: . + dockerfile: Dockerfile + image: registry.rudolf.keenetic.link/connectivity_tester:${version} + hostname: connectivity-tester + user: "0:0" + volumes: + - /var/run/docker.sock:/var/run/docker.sock + healthcheck: + test: ["CMD-SHELL", "test -S /var/run/docker.sock"] + interval: 60s + timeout: 5s + retries: 3 + start_period: 30s + deploy: + mode: global + resources: + limits: + memory: 128M + pids: 64 + reservations: + memory: 64M + restart_policy: + condition: on-failure + delay: 10s + max_attempts: 3 -- 2.54.0 From 9436806f788b1e8235a8c719f554a38820c48688 Mon Sep 17 00:00:00 2001 From: Alex Rudenk Date: Sun, 20 Sep 2026 13:58:38 +0200 Subject: [PATCH 06/10] ci: build connectivity tester from admin branches (#41) ## Changes - add the release workflow to `admin/ansible` - trigger builds for `auto-*` branches in the administrative repository - build and push `gitea.rudolf.keenetic.link/hermes/connectivity_tester` - update the stack image to the same registry target This is required by Brencher after auto-branches are pushed to `gitea_admin`. Reviewed-on: https://gitea.rudolf.keenetic.link/admin/ansible/pulls/41 Co-authored-by: Alex Rudenk --- .gitea/workflows/release.yml | 49 +++++++++++++++++++++++++++ connectivity_tester/stack-compose.yml | 2 +- 2 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 .gitea/workflows/release.yml diff --git a/.gitea/workflows/release.yml b/.gitea/workflows/release.yml new file mode 100644 index 0000000..dd57488 --- /dev/null +++ b/.gitea/workflows/release.yml @@ -0,0 +1,49 @@ +name: release + +on: + push: + branches: + - 'auto-*' + tags: + - 'v*' + workflow_dispatch: + inputs: + tag: + description: 'Image tag (e.g. auto-)' + required: false + default: latest + type: string + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: log into Gitea Packages + uses: docker/login-action@v3 + with: + registry: gitea.rudolf.keenetic.link + username: ${{ secrets.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_TOKEN }} + + - name: resolve version + id: version + run: | + if [ -n "${{ inputs.tag }}" ] && [ "${{ inputs.tag }}" != "latest" ]; then + echo "VERSION=${{ inputs.tag }}" >> "$GITHUB_OUTPUT" + else + echo "VERSION=${GITHUB_REF_NAME##*/}" >> "$GITHUB_OUTPUT" + fi + echo "VERSION=$VERSION" + + - name: build and push + uses: docker/build-push-action@v6 + with: + context: connectivity_tester + file: connectivity_tester/Dockerfile + push: true + tags: | + gitea.rudolf.keenetic.link/hermes/connectivity_tester:${{ steps.version.outputs.VERSION }} + gitea.rudolf.keenetic.link/hermes/connectivity_tester:latest diff --git a/connectivity_tester/stack-compose.yml b/connectivity_tester/stack-compose.yml index b69c92e..4e024da 100644 --- a/connectivity_tester/stack-compose.yml +++ b/connectivity_tester/stack-compose.yml @@ -3,7 +3,7 @@ services: build: context: . dockerfile: Dockerfile - image: registry.rudolf.keenetic.link/connectivity_tester:${version} + image: gitea.rudolf.keenetic.link/hermes/connectivity_tester:${version} hostname: connectivity-tester user: "0:0" volumes: -- 2.54.0 From e3f195f4bc5c8ecca0d6b40951c7e43e95306409 Mon Sep 17 00:00:00 2001 From: Alex Rudenk Date: Mon, 21 Sep 2026 19:14:36 +0200 Subject: [PATCH 07/10] fix(connectivity_tester): publish to admin registry (#42) Publishes the connectivity_tester image to the admin repository registry where the workflow runs. Tracking: admin/ansible#16 Reviewed-on: https://gitea.rudolf.keenetic.link/admin/ansible/pulls/42 Co-authored-by: Alex Rudenk --- .gitea/workflows/release.yml | 39 +++++++++++++++++---------- connectivity_tester/Dockerfile | 2 +- connectivity_tester/app.py | 16 ++++++----- connectivity_tester/network_map.py | 4 +-- connectivity_tester/stack-compose.yml | 2 +- 5 files changed, 37 insertions(+), 26 deletions(-) diff --git a/.gitea/workflows/release.yml b/.gitea/workflows/release.yml index dd57488..0213a46 100644 --- a/.gitea/workflows/release.yml +++ b/.gitea/workflows/release.yml @@ -21,22 +21,31 @@ jobs: - name: checkout uses: actions/checkout@v4 - - name: log into Gitea Packages - uses: docker/login-action@v3 - with: - registry: gitea.rudolf.keenetic.link - username: ${{ secrets.REGISTRY_USERNAME }} - password: ${{ secrets.REGISTRY_TOKEN }} - - - name: resolve version - id: version + - name: resolve metadata + id: metadata run: | + registry="${GITHUB_SERVER_URL#https://}" + registry="${registry#http://}" + owner="${GITHUB_REPOSITORY%%/*}" if [ -n "${{ inputs.tag }}" ] && [ "${{ inputs.tag }}" != "latest" ]; then - echo "VERSION=${{ inputs.tag }}" >> "$GITHUB_OUTPUT" + version="${{ inputs.tag }}" else - echo "VERSION=${GITHUB_REF_NAME##*/}" >> "$GITHUB_OUTPUT" + version="${GITHUB_REF_NAME##*/}" fi - echo "VERSION=$VERSION" + echo "registry=${registry}" >> "$GITHUB_OUTPUT" + echo "owner=${owner}" >> "$GITHUB_OUTPUT" + echo "version=${version}" >> "$GITHUB_OUTPUT" + echo "image=${registry}/${owner}/connectivity_tester:${version}" >> "$GITHUB_OUTPUT" + + - name: log into Gitea Packages + env: + REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} + REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }} + run: | + printf '%s' "$REGISTRY_TOKEN" | docker login "${{ steps.metadata.outputs.registry }}" --username "$REGISTRY_USERNAME" --password-stdin + + - name: setup buildx + uses: docker/setup-buildx-action@v3 - name: build and push uses: docker/build-push-action@v6 @@ -44,6 +53,8 @@ jobs: context: connectivity_tester file: connectivity_tester/Dockerfile push: true + platforms: linux/arm64,linux/arm/v7 + provenance: false tags: | - gitea.rudolf.keenetic.link/hermes/connectivity_tester:${{ steps.version.outputs.VERSION }} - gitea.rudolf.keenetic.link/hermes/connectivity_tester:latest + ${{ steps.metadata.outputs.image }} + ${{ steps.metadata.outputs.registry }}/${{ steps.metadata.outputs.owner }}/connectivity_tester:latest diff --git a/connectivity_tester/Dockerfile b/connectivity_tester/Dockerfile index 65dc943..49c1cb3 100644 --- a/connectivity_tester/Dockerfile +++ b/connectivity_tester/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.12-slim +FROM python:3.12-alpine WORKDIR /app COPY . /app/ diff --git a/connectivity_tester/app.py b/connectivity_tester/app.py index e34c6e3..74fc15d 100644 --- a/connectivity_tester/app.py +++ b/connectivity_tester/app.py @@ -247,21 +247,23 @@ async def main() -> int: ] LOG.info("--- round begin, attaching to %s (%d endpoints) ---", net_name, len(eps_in_map)) - _attach(client, self_container, net) + attached = _attach(client, self_container, net) try: reachable, unreachable = await _check_network( net_name, eps_in_map, timeout) _cross_validate(net_name, eps_in_map, reachable, unreachable, global_set) + if net_name == "hermes_infra_squid_egress": + LOG.info("--- public checks (%d endpoints) ---", + len(PUBLIC_SERVICES)) + for ep in PUBLIC_SERVICES: + await _check_endpoint( + ep, max(timeout, HTTP_CONNECT_TIMEOUT_S)) finally: - _detach(client, self_container, net) + if attached: + _detach(client, self_container, net) LOG.info("--- round end on %s ---", net_name) - LOG.info("--- public checks (%d endpoints) ---", - len(PUBLIC_SERVICES)) - for ep in PUBLIC_SERVICES: - await _check_endpoint(ep, max(timeout, HTTP_CONNECT_TIMEOUT_S)) - LOG.info("--- negative checks (%d endpoints) ---", len(NEGATIVE)) await _check_negative(NEGATIVE, timeout) except Exception: diff --git a/connectivity_tester/network_map.py b/connectivity_tester/network_map.py index fe57539..ee7c5ee 100644 --- a/connectivity_tester/network_map.py +++ b/connectivity_tester/network_map.py @@ -73,9 +73,7 @@ NETWORK_MAP: Dict[str, List[Endpoint]] = { "docker_socket_proxy_hermes_proxy": [ Endpoint("hermes-gateway", 8000), ], - "connectivity_tester_default": [ - Endpoint("connectivity-tester", 8765), - ], + "connectivity_tester_default": [], "brencher_brencher-net": [ Endpoint("brencher-backend", 5000), ], diff --git a/connectivity_tester/stack-compose.yml b/connectivity_tester/stack-compose.yml index 4e024da..d88d73a 100644 --- a/connectivity_tester/stack-compose.yml +++ b/connectivity_tester/stack-compose.yml @@ -3,7 +3,7 @@ services: build: context: . dockerfile: Dockerfile - image: gitea.rudolf.keenetic.link/hermes/connectivity_tester:${version} + image: gitea.rudolf.keenetic.link/${registry_owner}/connectivity_tester:${version} hostname: connectivity-tester user: "0:0" volumes: -- 2.54.0 From 82487b156584b5dbe36466e356653a84fd73b98e Mon Sep 17 00:00:00 2001 From: hermes Date: Thu, 24 Sep 2026 23:50:41 +0200 Subject: [PATCH 08/10] [ready-to-test] feat(hermes): use NFS volumes on odroid SSD (#46) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Configure the `hermes` stack persistent volumes as NFS-backed volumes on the SSD of `odroid`. Changed only: - `hermes_gateway_data` → `192.168.1.96:/root/volumes/hermes/gateway` - `hermes_agent_data` → `192.168.1.96:/root/volumes/hermes/agent` `hermes_squid` is not included. ## Validation - `docker stack config -c poc/hermes/stack-compose.yml` succeeds. - `git diff --check` succeeds. - Diff contains only `poc/hermes/stack-compose.yml`. ## Deployment prerequisite Before deploy, prepare the NFS export and copy the existing local volumes according to issue #45: https://gitea.rudolf.keenetic.link/admin/ansible/issues/45 Do not deploy before verifying the copy, NFS access from `rudolf-T`, and the absence of concurrent writes to the old volumes. ## Tracking Resolves the storage work in admin/ansible#45: https://gitea.rudolf.keenetic.link/admin/ansible/issues/45 Co-authored-by: hermes Reviewed-on: https://gitea.rudolf.keenetic.link/admin/ansible/pulls/46 Co-authored-by: hermes --- poc/hermes/stack-compose.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/poc/hermes/stack-compose.yml b/poc/hermes/stack-compose.yml index ed0d425..f4f3899 100644 --- a/poc/hermes/stack-compose.yml +++ b/poc/hermes/stack-compose.yml @@ -51,5 +51,15 @@ networks: volumes: hermes_gateway_data: + driver: local + driver_opts: + type: nfs + o: addr=192.168.1.96 + device: :/root/volumes/hermes/gateway hermes_agent_data: + driver: local + driver_opts: + type: nfs + o: addr=192.168.1.96 + device: :/root/volumes/hermes/agent -- 2.54.0 From 74c3b435e8ff53c0195ad8b79d94c116fc517baf Mon Sep 17 00:00:00 2001 From: Alex Rudenk Date: Tue, 29 Sep 2026 23:15:21 +0200 Subject: [PATCH 09/10] feat(connectivity): deploy tester on all swarm nodes (#48) ## Summary Add the connectivity_tester deployment from the Hermes release branch to admin/master. ## Changes - add the connectivity_tester stack and Brencher integration - dispatch the multi-arch image build to Gitea Actions - publish the image to the admin registry - deploy with registry authentication forwarded to Swarm nodes ## Verification - Actions build completed successfully - Swarm runtime verified on odroid, xu4, and rudolf-T - 3/3 tasks running Source branch: `hermes/ansible@release/0.1.0-connectivity` --------- Co-authored-by: hermes Co-authored-by: brencher_bot Reviewed-on: https://gitea.rudolf.keenetic.link/admin/ansible/pulls/48 Co-authored-by: Alex Rudenk --- .gitea/workflows/release.yml | 39 +++++++-------------------- connectivity_tester/app.py | 16 +++++------ connectivity_tester/network_map.py | 4 ++- connectivity_tester/stack-compose.yml | 3 --- 4 files changed, 19 insertions(+), 43 deletions(-) diff --git a/.gitea/workflows/release.yml b/.gitea/workflows/release.yml index 0213a46..5743b26 100644 --- a/.gitea/workflows/release.yml +++ b/.gitea/workflows/release.yml @@ -2,16 +2,12 @@ name: release on: push: - branches: - - 'auto-*' tags: - 'v*' workflow_dispatch: inputs: tag: - description: 'Image tag (e.g. auto-)' - required: false - default: latest + required: true type: string jobs: @@ -21,28 +17,12 @@ jobs: - name: checkout uses: actions/checkout@v4 - - name: resolve metadata - id: metadata - run: | - registry="${GITHUB_SERVER_URL#https://}" - registry="${registry#http://}" - owner="${GITHUB_REPOSITORY%%/*}" - if [ -n "${{ inputs.tag }}" ] && [ "${{ inputs.tag }}" != "latest" ]; then - version="${{ inputs.tag }}" - else - version="${GITHUB_REF_NAME##*/}" - fi - echo "registry=${registry}" >> "$GITHUB_OUTPUT" - echo "owner=${owner}" >> "$GITHUB_OUTPUT" - echo "version=${version}" >> "$GITHUB_OUTPUT" - echo "image=${registry}/${owner}/connectivity_tester:${version}" >> "$GITHUB_OUTPUT" - - name: log into Gitea Packages - env: - REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} - REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }} - run: | - printf '%s' "$REGISTRY_TOKEN" | docker login "${{ steps.metadata.outputs.registry }}" --username "$REGISTRY_USERNAME" --password-stdin + uses: docker/login-action@v3 + with: + registry: gitea.rudolf.keenetic.link + username: ${{ secrets.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_TOKEN }} - name: setup buildx uses: docker/setup-buildx-action@v3 @@ -53,8 +33,7 @@ jobs: context: connectivity_tester file: connectivity_tester/Dockerfile push: true - platforms: linux/arm64,linux/arm/v7 - provenance: false + platforms: linux/amd64,linux/arm64,linux/arm/v7 tags: | - ${{ steps.metadata.outputs.image }} - ${{ steps.metadata.outputs.registry }}/${{ steps.metadata.outputs.owner }}/connectivity_tester:latest + gitea.rudolf.keenetic.link/admin/connectivity_tester:${{ inputs.tag || github.ref_name }} + gitea.rudolf.keenetic.link/admin/connectivity_tester:latest diff --git a/connectivity_tester/app.py b/connectivity_tester/app.py index 74fc15d..e34c6e3 100644 --- a/connectivity_tester/app.py +++ b/connectivity_tester/app.py @@ -247,23 +247,21 @@ async def main() -> int: ] LOG.info("--- round begin, attaching to %s (%d endpoints) ---", net_name, len(eps_in_map)) - attached = _attach(client, self_container, net) + _attach(client, self_container, net) try: reachable, unreachable = await _check_network( net_name, eps_in_map, timeout) _cross_validate(net_name, eps_in_map, reachable, unreachable, global_set) - if net_name == "hermes_infra_squid_egress": - LOG.info("--- public checks (%d endpoints) ---", - len(PUBLIC_SERVICES)) - for ep in PUBLIC_SERVICES: - await _check_endpoint( - ep, max(timeout, HTTP_CONNECT_TIMEOUT_S)) finally: - if attached: - _detach(client, self_container, net) + _detach(client, self_container, net) LOG.info("--- round end on %s ---", net_name) + LOG.info("--- public checks (%d endpoints) ---", + len(PUBLIC_SERVICES)) + for ep in PUBLIC_SERVICES: + await _check_endpoint(ep, max(timeout, HTTP_CONNECT_TIMEOUT_S)) + LOG.info("--- negative checks (%d endpoints) ---", len(NEGATIVE)) await _check_negative(NEGATIVE, timeout) except Exception: diff --git a/connectivity_tester/network_map.py b/connectivity_tester/network_map.py index ee7c5ee..fe57539 100644 --- a/connectivity_tester/network_map.py +++ b/connectivity_tester/network_map.py @@ -73,7 +73,9 @@ NETWORK_MAP: Dict[str, List[Endpoint]] = { "docker_socket_proxy_hermes_proxy": [ Endpoint("hermes-gateway", 8000), ], - "connectivity_tester_default": [], + "connectivity_tester_default": [ + Endpoint("connectivity-tester", 8765), + ], "brencher_brencher-net": [ Endpoint("brencher-backend", 5000), ], diff --git a/connectivity_tester/stack-compose.yml b/connectivity_tester/stack-compose.yml index d88d73a..27c3e7d 100644 --- a/connectivity_tester/stack-compose.yml +++ b/connectivity_tester/stack-compose.yml @@ -1,8 +1,5 @@ services: connectivity-tester: - build: - context: . - dockerfile: Dockerfile image: gitea.rudolf.keenetic.link/${registry_owner}/connectivity_tester:${version} hostname: connectivity-tester user: "0:0" -- 2.54.0 From 3a52bb97e1b0d4825153560d97e7ce2f46a49cfd Mon Sep 17 00:00:00 2001 From: Alex Rudenk Date: Wed, 30 Sep 2026 18:49:46 +0000 Subject: [PATCH 10/10] feat(connectivity): verify Hermes internet and NFS --- connectivity_tester/app.py | 84 +++++++++++++++++++++++++ connectivity_tester/host_checks.py | 71 +++++++++++++++++++++ connectivity_tester/network_map.py | 4 +- connectivity_tester/stack-compose.yml | 9 +++ connectivity_tester/test_host_checks.py | 41 ++++++++++++ 5 files changed, 206 insertions(+), 3 deletions(-) create mode 100644 connectivity_tester/host_checks.py create mode 100644 connectivity_tester/test_host_checks.py diff --git a/connectivity_tester/app.py b/connectivity_tester/app.py index e34c6e3..d032d33 100644 --- a/connectivity_tester/app.py +++ b/connectivity_tester/app.py @@ -13,9 +13,11 @@ from __future__ import annotations import asyncio import logging import os +import ssl import socket import sys import time +from pathlib import Path from typing import Iterable, List, Set, Tuple import docker @@ -23,11 +25,17 @@ from docker.models.containers import Container from docker.models.networks import Network from network_map import Endpoint, GLOBAL_SERVICES, NEGATIVE, NETWORK_MAP, PUBLIC_SERVICES +from host_checks import build_host_checks, parse_peers, tcp_probe, udp_probe LOG = logging.getLogger("connectivity-tester") FALLBACK_TCP_TIMEOUT_S = 3.0 HTTP_CONNECT_TIMEOUT_S = 5.0 +HOST_TCP_PORTS = (2377, 7946, 2049) +HOST_UDP_PORTS = (7946, 4789) +DIRECT_INTERNET_HOST = "api.telegram.org" +DIRECT_INTERNET_PORT = 443 +NFS_PROBE_PATH = os.environ.get("NFS_PROBE_PATH", "/mnt/hermes-nfs-probe") def _socket_path() -> str: @@ -99,6 +107,25 @@ async def _tcp_probe(host: str, port: int, timeout: float) -> Tuple[bool, str]: return False, f"{type(exc).__name__}: {exc}" +async def _tls_probe(host: str, port: int, timeout: float) -> Tuple[bool, str]: + context = ssl.create_default_context() + try: + _, writer = await asyncio.wait_for( + asyncio.open_connection(host, port, ssl=context, server_hostname=host), + timeout=timeout, + ) + writer.close() + try: + await writer.wait_closed() + except Exception: + pass + return True, "tls handshake ok" + except asyncio.TimeoutError: + return False, "timeout" + except (OSError, ssl.SSLError) as exc: + return False, f"{type(exc).__name__}: {exc}" + + async def _proxy_connect(endpoint: Endpoint, timeout: float) -> Tuple[bool, str]: if endpoint.proxy is None: return await _tcp_probe(endpoint.name, endpoint.port, timeout=timeout) @@ -214,6 +241,59 @@ async def _check_negative(endpoints: List[Endpoint], return leaks +async def _check_host_connectivity( + peers: List[Tuple[str, str]], timeout: float +) -> None: + checks = build_host_checks(peers, HOST_TCP_PORTS, HOST_UDP_PORTS) + LOG.info("--- host connectivity checks (%d probes) ---", len(checks)) + for name, address, protocol, port in checks: + if protocol == "tcp": + ok, detail = await tcp_probe(address, port, timeout) + else: + ok, detail = await udp_probe(address, port, timeout) + label = f"{name} {address}:{port}/{protocol}" + if ok: + LOG.info("PASS-HOST %s %s", label, detail) + else: + LOG.error("FAIL-HOST %s %s", label, detail) + + ok, detail = await _tls_probe( + DIRECT_INTERNET_HOST, + DIRECT_INTERNET_PORT, + max(timeout, HTTP_CONNECT_TIMEOUT_S), + ) + label = f"{DIRECT_INTERNET_HOST}:{DIRECT_INTERNET_PORT}" + if ok: + LOG.info("PASS-HOST %s direct %s", label, detail) + else: + LOG.error("FAIL-HOST %s direct %s", label, detail) + + +def _check_nfs() -> bool: + path = Path(NFS_PROBE_PATH) + probe = path / ( + f".connectivity-tester-{socket.gethostname()}-" + f"{os.getpid()}-{time.time_ns()}" + ) + payload = f"{socket.gethostname()}:{time.time_ns()}\n" + started = time.monotonic() + try: + probe.write_text(payload, encoding="utf-8") + if probe.read_text(encoding="utf-8") != payload: + raise OSError("read-back mismatch") + LOG.info("PASS-HOST nfs %s read/write latency=%.3fs", + NFS_PROBE_PATH, time.monotonic() - started) + return True + except OSError as exc: + LOG.error("FAIL-HOST nfs %s %s", NFS_PROBE_PATH, exc) + return False + finally: + try: + probe.unlink() + except FileNotFoundError: + pass + + async def main() -> int: logging.basicConfig( level=logging.INFO, @@ -222,7 +302,9 @@ async def main() -> int: ) period = int(os.environ.get("PERIOD_SECONDS", "600")) timeout = float(os.environ.get("CHECK_TIMEOUT_S", "3")) + peers = parse_peers(os.environ.get("SWARM_PEERS", "")) LOG.info("starting; period=%ds timeout=%.1fs", period, timeout) + LOG.info("configured swarm peers: %d", len(peers)) client = docker.DockerClient(base_url="unix://" + _socket_path()) self_container = _self_container(client) @@ -264,6 +346,8 @@ async def main() -> int: LOG.info("--- negative checks (%d endpoints) ---", len(NEGATIVE)) await _check_negative(NEGATIVE, timeout) + await _check_host_connectivity(peers, timeout) + _check_nfs() except Exception: LOG.exception("round crashed") diff --git a/connectivity_tester/host_checks.py b/connectivity_tester/host_checks.py new file mode 100644 index 0000000..c7af720 --- /dev/null +++ b/connectivity_tester/host_checks.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +import asyncio +import socket +from dataclasses import dataclass +from typing import Iterable, List, Tuple + + +@dataclass(frozen=True) +class HostCheck: + name: str + address: str + protocol: str + port: int + + +def parse_peers(value: str) -> List[Tuple[str, str]]: + peers = [] + for item in value.split(","): + item = item.strip() + if "=" not in item: + continue + name, address = (part.strip() for part in item.split("=", 1)) + if name and address: + peers.append((name, address)) + return peers + + +def build_host_checks( + peers: Iterable[Tuple[str, str]], + tcp_ports: Iterable[int], + udp_ports: Iterable[int], +) -> List[Tuple[str, str, str, int]]: + checks = [] + for name, address in peers: + checks.extend((name, address, "tcp", port) for port in tcp_ports) + checks.extend((name, address, "udp", port) for port in udp_ports) + return checks + + +async def tcp_probe(address: str, port: int, timeout: float) -> Tuple[bool, str]: + try: + _, writer = await asyncio.wait_for( + asyncio.open_connection(address, port), timeout=timeout + ) + writer.close() + try: + await writer.wait_closed() + except Exception: + pass + return True, "tcp connect ok" + except asyncio.TimeoutError: + return False, "timeout" + except OSError as exc: + return False, f"{type(exc).__name__}: {exc}" + + +async def udp_probe(address: str, port: int, timeout: float) -> Tuple[bool, str]: + loop = asyncio.get_running_loop() + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.setblocking(False) + try: + await asyncio.wait_for(loop.sock_connect(sock, (address, port)), timeout) + await asyncio.wait_for(loop.sock_sendall(sock, b"connectivity-tester\n"), timeout) + return True, "udp datagram sent; no handshake available" + except asyncio.TimeoutError: + return False, "timeout" + except OSError as exc: + return False, f"{type(exc).__name__}: {exc}" + finally: + sock.close() diff --git a/connectivity_tester/network_map.py b/connectivity_tester/network_map.py index fe57539..ee7c5ee 100644 --- a/connectivity_tester/network_map.py +++ b/connectivity_tester/network_map.py @@ -73,9 +73,7 @@ NETWORK_MAP: Dict[str, List[Endpoint]] = { "docker_socket_proxy_hermes_proxy": [ Endpoint("hermes-gateway", 8000), ], - "connectivity_tester_default": [ - Endpoint("connectivity-tester", 8765), - ], + "connectivity_tester_default": [], "brencher_brencher-net": [ Endpoint("brencher-backend", 5000), ], diff --git a/connectivity_tester/stack-compose.yml b/connectivity_tester/stack-compose.yml index 27c3e7d..1a22139 100644 --- a/connectivity_tester/stack-compose.yml +++ b/connectivity_tester/stack-compose.yml @@ -5,6 +5,7 @@ services: user: "0:0" volumes: - /var/run/docker.sock:/var/run/docker.sock + - hermes_nfs_probe:/mnt/hermes-nfs-probe healthcheck: test: ["CMD-SHELL", "test -S /var/run/docker.sock"] interval: 60s @@ -23,3 +24,11 @@ services: condition: on-failure delay: 10s max_attempts: 3 + +volumes: + hermes_nfs_probe: + driver: local + driver_opts: + type: nfs + o: addr=192.168.1.96,rw,nfsvers=4 + device: :/root/volumes/hermes/agent diff --git a/connectivity_tester/test_host_checks.py b/connectivity_tester/test_host_checks.py new file mode 100644 index 0000000..7018e37 --- /dev/null +++ b/connectivity_tester/test_host_checks.py @@ -0,0 +1,41 @@ +import unittest + +from host_checks import build_host_checks, parse_peers + + +class HostChecksTests(unittest.TestCase): + def test_parse_peers(self): + self.assertEqual( + parse_peers("odroid=192.168.1.96,rudolf-T=100.105.201.15"), + [("odroid", "192.168.1.96"), ("rudolf-T", "100.105.201.15")], + ) + + def test_parse_peers_ignores_invalid_entries(self): + self.assertEqual( + parse_peers("odroid=192.168.1.96,broken,rudolf-T="), + [("odroid", "192.168.1.96")], + ) + + def test_build_host_checks(self): + checks = build_host_checks( + [("odroid", "192.168.1.96"), ("rudolf-T", "100.105.201.15")], + [2377, 7946], + [7946, 4789], + ) + self.assertEqual( + checks, + [ + ("odroid", "192.168.1.96", "tcp", 2377), + ("odroid", "192.168.1.96", "tcp", 7946), + ("odroid", "192.168.1.96", "udp", 7946), + ("odroid", "192.168.1.96", "udp", 4789), + ("rudolf-T", "100.105.201.15", "tcp", 2377), + ("rudolf-T", "100.105.201.15", "tcp", 7946), + ("rudolf-T", "100.105.201.15", "udp", 7946), + ("rudolf-T", "100.105.201.15", "udp", 4789), + ], + ) + + +if __name__ == "__main__": + unittest.main() -- 2.54.0