Compare commits

..
Author SHA1 Message Date
hermes 6848d114a7 chore(hermes_squid): remove brencher-state-proxy
Drop the brencher-state-proxy sidecar from hermes_infra -- it is
no longer needed. The sidecar existed to relay /state from
brencher.rudolf.keenetic.link into the admin overlay; that relay is
unnecessary since brencher ingress handles the path itself.

Changes vs the brencher-state-proxy commit that was merged in #4:
- delete hermes_squid/brencher-state-proxy.conf
- remove the brencher-state-proxy: service block from
  hermes_squid/stack-compose.yml
- remove the brencher_state_proxy_conf: top-level config object
- drop brencher-state-proxy from the NO_PROXY / no_proxy lists
- update the surrounding comments
2026-08-29 22:47:36 +00:00
28 changed files with 31 additions and 1629 deletions
-39
View File
@@ -1,39 +0,0 @@
name: release
on:
push:
tags:
- 'v*'
workflow_dispatch:
inputs:
tag:
required: true
type: string
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: checkout
uses: actions/checkout@v4
- name: log into Gitea Packages
uses: docker/login-action@v3
with:
registry: gitea.rudolf.keenetic.link
username: ${{ secrets.REGISTRY_USERNAME }}
password: ${{ secrets.REGISTRY_TOKEN }}
- name: setup buildx
uses: docker/setup-buildx-action@v3
- name: build and push
uses: docker/build-push-action@v6
with:
context: connectivity_tester
file: connectivity_tester/Dockerfile
push: true
platforms: linux/amd64,linux/arm64,linux/arm/v7
tags: |
gitea.rudolf.keenetic.link/admin/connectivity_tester:${{ inputs.tag || github.ref_name }}
gitea.rudolf.keenetic.link/admin/connectivity_tester:latest
+1 -3
View File
@@ -4,9 +4,7 @@
docker swarm init --advertise-addr 192.168.1.96 --force-new-cluster
Swarm initialized: current node (j9bljgaqh9qch2nmozvu2bprs) is now a manager.
To add a worker to this swarm, run the following command:
docker swarm join --token <token> \
--advertise-addr 100.105.201.15 192.168.1.96:2377
docker swarm join --token SWMTKN-1-0eb9ypg3ouig0ac493scmjtogida7ddy79xr2e9kkvaitoz0y0-7t5lg5sx8q20uqz2uandsblc9 192.168.1.96:2377
## Prod instances
-12
View File
@@ -1,12 +0,0 @@
FROM python:3.12-alpine
WORKDIR /app
COPY . /app/
RUN pip install --no-cache-dir docker==7.1.0
ENV PYTHONUNBUFFERED=1 \
PERIOD_SECONDS=600 \
CHECK_TIMEOUT_S=3
CMD ["python", "-m", "app"]
-360
View File
@@ -1,360 +0,0 @@
"""Connectivity tester.
For every configured overlay network the container attaches itself, runs
TCP checks against its endpoints, then detaches. Endpoints marked with
``proxy=(host, port)`` are probed through that HTTP proxy (CONNECT). After
each round a final loop checks PUBLIC_SERVICES. Cross-validation:
- service reachable but not in map -> ERROR 'LEAK-UNEXPECTED'
- service in map but unreachable -> ERROR 'MISSING-EXPECTED'
- global service not in any map -> WARNING 'GLOBAL-UNCOVERED'
"""
from __future__ import annotations
import asyncio
import logging
import os
import ssl
import socket
import sys
import time
from pathlib import Path
from typing import Iterable, List, Set, Tuple
import docker
from docker.models.containers import Container
from docker.models.networks import Network
from network_map import Endpoint, GLOBAL_SERVICES, NEGATIVE, NETWORK_MAP, PUBLIC_SERVICES
from host_checks import build_host_checks, parse_peers, tcp_probe, udp_probe
LOG = logging.getLogger("connectivity-tester")
FALLBACK_TCP_TIMEOUT_S = 3.0
HTTP_CONNECT_TIMEOUT_S = 5.0
HOST_TCP_PORTS = (2377, 7946, 2049)
HOST_UDP_PORTS = (7946, 4789)
DIRECT_INTERNET_HOST = "api.telegram.org"
DIRECT_INTERNET_PORT = 443
NFS_PROBE_PATH = os.environ.get("NFS_PROBE_PATH", "/mnt/hermes-nfs-probe")
def _socket_path() -> str:
return os.environ.get("DOCKER_SOCKET", "/var/run/docker.sock")
def _self_container(client: docker.DockerClient) -> Container:
hostname = socket.gethostname()
for c in client.containers.list():
cn = c.name.lstrip("/")
hn = c.attrs.get("Config", {}).get("Hostname", "")
if hostname == cn or hostname == hn or hostname[:12] == c.id[:12]:
return c
candidates = [c for c in client.containers.list() if hostname[:12] == c.id]
if len(candidates) == 1:
return candidates[0]
raise RuntimeError(f"self container not found for hostname={hostname}")
def _swarm_networks_for(client: docker.DockerClient,
names: Iterable[str]) -> dict:
out = {}
by_name = {n.name: n for n in client.networks.list()}
for name in names:
net = by_name.get(name)
if net is not None and net.attrs.get("Scope") == "swarm":
out[name] = net
return out
def _attach(client: docker.DockerClient, container: Container,
network: Network) -> bool:
current = (container.attrs.get("NetworkSettings", {}) or {}).get("Networks", {})
if network.name in current:
return False
try:
network.connect(container, aliases=["connectivity-tester",
socket.gethostname()])
return True
except Exception as exc:
if "already exists" in str(exc).lower():
return False
raise
def _detach(client: docker.DockerClient, container: Container,
network: Network) -> None:
try:
network.disconnect(container, force=True)
except Exception as exc:
if "not found" in str(exc).lower():
return
raise
async def _tcp_probe(host: str, port: int, timeout: float) -> Tuple[bool, str]:
try:
fut = asyncio.open_connection(host=host, port=port)
reader, writer = await asyncio.wait_for(fut, timeout=timeout)
writer.close()
try:
await writer.wait_closed()
except Exception:
pass
return True, "tcp ok"
except asyncio.TimeoutError:
return False, "timeout"
except OSError as exc:
return False, f"{type(exc).__name__}: {exc}"
async def _tls_probe(host: str, port: int, timeout: float) -> Tuple[bool, str]:
context = ssl.create_default_context()
try:
_, writer = await asyncio.wait_for(
asyncio.open_connection(host, port, ssl=context, server_hostname=host),
timeout=timeout,
)
writer.close()
try:
await writer.wait_closed()
except Exception:
pass
return True, "tls handshake ok"
except asyncio.TimeoutError:
return False, "timeout"
except (OSError, ssl.SSLError) as exc:
return False, f"{type(exc).__name__}: {exc}"
async def _proxy_connect(endpoint: Endpoint, timeout: float) -> Tuple[bool, str]:
if endpoint.proxy is None:
return await _tcp_probe(endpoint.name, endpoint.port, timeout=timeout)
proxy_host, proxy_port = endpoint.proxy
target = endpoint.name
target_port = endpoint.port
try:
fut = asyncio.open_connection(host=proxy_host, port=proxy_port)
reader, writer = await asyncio.wait_for(fut, timeout=timeout)
req = (
f"CONNECT {target}:{target_port} HTTP/1.1\r\n"
f"Host: {target}:{target_port}\r\n\r\n"
)
writer.write(req.encode("ascii"))
await writer.drain()
status_line = await asyncio.wait_for(reader.readline(),
timeout=timeout)
if not status_line.startswith(b"HTTP/") or b" 200 " not in status_line:
writer.close()
try:
await writer.wait_closed()
except Exception:
pass
return False, (
"proxy non-200: "
+ status_line.strip().decode(errors="replace")
)
while True:
line = await asyncio.wait_for(reader.readline(), timeout=timeout)
if line in (b"\r\n", b"", b"\n"):
break
writer.close()
try:
await writer.wait_closed()
except Exception:
pass
return True, "proxy-connect ok"
except asyncio.TimeoutError:
return False, "timeout"
except OSError as exc:
return False, f"{type(exc).__name__}: {exc}"
async def _check_endpoint(endpoint: Endpoint, timeout: float) -> bool:
if endpoint.proxy is not None:
ok, detail = await _proxy_connect(endpoint, timeout)
else:
ok, detail = await _tcp_probe(endpoint.name, endpoint.port,
timeout=timeout)
label = f"{endpoint.name}:{endpoint.port}"
src = f"via {endpoint.proxy}" if endpoint.proxy else "direct"
if ok:
LOG.info("PASS %s %s %s", label, src, detail)
else:
LOG.error("FAIL %s %s %s", label, src, detail)
return ok
async def _check_network(network_name: str,
endpoints: List[Endpoint],
timeout: float) -> Tuple[Set[Endpoint], Set[Endpoint]]:
reachable: Set[Endpoint] = set()
unreachable: Set[Endpoint] = set()
for ep in endpoints:
ok = await _check_endpoint(ep, timeout)
(reachable if ok else unreachable).add(ep)
return reachable, unreachable
def _cross_validate(network_name: str,
mapped: List[Endpoint],
reachable: Set[Endpoint],
unreachable: Set[Endpoint],
global_set: Set[Endpoint]) -> None:
mapped_set = set(mapped)
reachable_global = reachable & global_set
for ep in reachable_global - mapped_set:
LOG.error("LEAK-UNEXPECTED %s reachable in %s but not in map",
f"{ep.name}:{ep.port}", network_name)
for ep in mapped_set & unreachable:
LOG.error("MISSING-EXPECTED %s unreachable in %s but listed in map",
f"{ep.name}:{ep.port}", network_name)
for ep in mapped_set & reachable:
LOG.info("OK-MAPPED %s ok in %s", f"{ep.name}:{ep.port}", network_name)
for ep in mapped_set - reachable - unreachable:
LOG.warning("NOT-CHECKED %s in map for %s",
f"{ep.name}:{ep.port}", network_name)
def _consistency_audit(net_names: List[str]) -> None:
flat: Set[Endpoint] = set()
for n in net_names:
flat.update(NETWORK_MAP.get(n, []))
covered = flat | set(PUBLIC_SERVICES)
all_global = set(GLOBAL_SERVICES)
uncovered = all_global - covered
if uncovered:
LOG.warning("GLOBAL-UNCOVERED %s not present in any map",
sorted(f"{e.name}:{e.port}" for e in uncovered))
async def _check_negative(endpoints: List[Endpoint],
timeout: float) -> int:
leaks = 0
for ep in endpoints:
ok = await _tcp_probe(ep.name, ep.port, timeout=timeout)
if ok:
LOG.error("LEAK %s:%d reachable but must NOT be",
ep.name, ep.port)
leaks += 1
else:
LOG.info("OK-NEG %s:%d blocked", ep.name, ep.port)
return leaks
async def _check_host_connectivity(
peers: List[Tuple[str, str]], timeout: float
) -> None:
checks = build_host_checks(peers, HOST_TCP_PORTS, HOST_UDP_PORTS)
LOG.info("--- host connectivity checks (%d probes) ---", len(checks))
for name, address, protocol, port in checks:
if protocol == "tcp":
ok, detail = await tcp_probe(address, port, timeout)
else:
ok, detail = await udp_probe(address, port, timeout)
label = f"{name} {address}:{port}/{protocol}"
if ok:
LOG.info("PASS-HOST %s %s", label, detail)
else:
LOG.error("FAIL-HOST %s %s", label, detail)
ok, detail = await _tls_probe(
DIRECT_INTERNET_HOST,
DIRECT_INTERNET_PORT,
max(timeout, HTTP_CONNECT_TIMEOUT_S),
)
label = f"{DIRECT_INTERNET_HOST}:{DIRECT_INTERNET_PORT}"
if ok:
LOG.info("PASS-HOST %s direct %s", label, detail)
else:
LOG.error("FAIL-HOST %s direct %s", label, detail)
def _check_nfs() -> bool:
path = Path(NFS_PROBE_PATH)
probe = path / (
f".connectivity-tester-{socket.gethostname()}-"
f"{os.getpid()}-{time.time_ns()}"
)
payload = f"{socket.gethostname()}:{time.time_ns()}\n"
started = time.monotonic()
try:
probe.write_text(payload, encoding="utf-8")
if probe.read_text(encoding="utf-8") != payload:
raise OSError("read-back mismatch")
LOG.info("PASS-HOST nfs %s read/write latency=%.3fs",
NFS_PROBE_PATH, time.monotonic() - started)
return True
except OSError as exc:
LOG.error("FAIL-HOST nfs %s %s", NFS_PROBE_PATH, exc)
return False
finally:
try:
probe.unlink()
except FileNotFoundError:
pass
async def main() -> int:
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(levelname)s %(message)s",
stream=sys.stdout,
)
period = int(os.environ.get("PERIOD_SECONDS", "600"))
timeout = float(os.environ.get("CHECK_TIMEOUT_S", "3"))
peers = parse_peers(os.environ.get("SWARM_PEERS", ""))
LOG.info("starting; period=%ds timeout=%.1fs", period, timeout)
LOG.info("configured swarm peers: %d", len(peers))
client = docker.DockerClient(base_url="unix://" + _socket_path())
self_container = _self_container(client)
nets = _swarm_networks_for(client, NETWORK_MAP.keys())
planned = list(NETWORK_MAP.keys())
LOG.info("configured networks: %d, present in swarm: %d",
len(planned), len(nets))
for n in nets:
LOG.info("plan network: %s", n)
_consistency_audit(planned)
global_set: Set[Endpoint] = set(GLOBAL_SERVICES)
while True:
started = time.monotonic()
try:
for net_name, net in nets.items():
eps_in_map: List[Endpoint] = [
e for e in NETWORK_MAP[net_name] if e.proxy is None
]
LOG.info("--- round begin, attaching to %s (%d endpoints) ---",
net_name, len(eps_in_map))
_attach(client, self_container, net)
try:
reachable, unreachable = await _check_network(
net_name, eps_in_map, timeout)
_cross_validate(net_name, eps_in_map,
reachable, unreachable, global_set)
finally:
_detach(client, self_container, net)
LOG.info("--- round end on %s ---", net_name)
LOG.info("--- public checks (%d endpoints) ---",
len(PUBLIC_SERVICES))
for ep in PUBLIC_SERVICES:
await _check_endpoint(ep, max(timeout, HTTP_CONNECT_TIMEOUT_S))
LOG.info("--- negative checks (%d endpoints) ---", len(NEGATIVE))
await _check_negative(NEGATIVE, timeout)
await _check_host_connectivity(peers, timeout)
_check_nfs()
except Exception:
LOG.exception("round crashed")
elapsed = time.monotonic() - started
sleep_for = max(0.0, period - elapsed)
await asyncio.sleep(sleep_for)
if __name__ == "__main__":
asyncio.run(main())
-71
View File
@@ -1,71 +0,0 @@
from __future__ import annotations
import asyncio
import socket
from dataclasses import dataclass
from typing import Iterable, List, Tuple
@dataclass(frozen=True)
class HostCheck:
name: str
address: str
protocol: str
port: int
def parse_peers(value: str) -> List[Tuple[str, str]]:
peers = []
for item in value.split(","):
item = item.strip()
if "=" not in item:
continue
name, address = (part.strip() for part in item.split("=", 1))
if name and address:
peers.append((name, address))
return peers
def build_host_checks(
peers: Iterable[Tuple[str, str]],
tcp_ports: Iterable[int],
udp_ports: Iterable[int],
) -> List[Tuple[str, str, str, int]]:
checks = []
for name, address in peers:
checks.extend((name, address, "tcp", port) for port in tcp_ports)
checks.extend((name, address, "udp", port) for port in udp_ports)
return checks
async def tcp_probe(address: str, port: int, timeout: float) -> Tuple[bool, str]:
try:
_, writer = await asyncio.wait_for(
asyncio.open_connection(address, port), timeout=timeout
)
writer.close()
try:
await writer.wait_closed()
except Exception:
pass
return True, "tcp connect ok"
except asyncio.TimeoutError:
return False, "timeout"
except OSError as exc:
return False, f"{type(exc).__name__}: {exc}"
async def udp_probe(address: str, port: int, timeout: float) -> Tuple[bool, str]:
loop = asyncio.get_running_loop()
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.setblocking(False)
try:
await asyncio.wait_for(loop.sock_connect(sock, (address, port)), timeout)
await asyncio.wait_for(loop.sock_sendall(sock, b"connectivity-tester\n"), timeout)
return True, "udp datagram sent; no handshake available"
except asyncio.TimeoutError:
return False, "timeout"
except OSError as exc:
return False, f"{type(exc).__name__}: {exc}"
finally:
sock.close()
-118
View File
@@ -1,118 +0,0 @@
from dataclasses import dataclass
from typing import Dict, List, Optional, Tuple
@dataclass(frozen=True)
class Endpoint:
name: str
port: int
proxy: Optional[Tuple[str, int]] = None
tls: bool = False
SWARM_NETWORKS = (
"hermes_infra_hermes_proxy",
"hermes_infra_squid_egress",
"hermes_infra_admin",
"hermes_infra_default",
"hermes_hermes",
"gitea_gitea",
"gmail_mcp_default",
"immich_default",
"torrserv_proxy_default",
"docker_socket_proxy_hermes_proxy",
"brencher_brencher-net",
"brencher2_brencher-net",
)
SQUID_PROXY: Tuple[str, int] = ("squid", 3128)
NETWORK_MAP: Dict[str, List[Endpoint]] = {
"hermes_infra_hermes_proxy": [
Endpoint("hermes-gateway", 8000),
Endpoint("hermes-ui", 8087),
Endpoint("squid", 3128),
Endpoint("docker_socket_proxy", 2375),
Endpoint("brencher-state-proxy", 80),
],
"hermes_infra_squid_egress": [
Endpoint("squid", 3128),
],
"hermes_infra_admin": [
Endpoint("sarg", 80),
],
"hermes_infra_default": [
Endpoint("hermes-ui", 8087),
],
"hermes_hermes": [
Endpoint("hermes-agent", 8000),
Endpoint("hermes-gateway", 8000),
],
"gitea_gitea": [
Endpoint("gitea-server", 3000),
Endpoint("act_runner", 8088),
Endpoint("cache_server", 37051),
],
"gmail_mcp_default": [
Endpoint("gmail-mcp-server", 8000),
Endpoint("telegram-mcp-server", 8000),
],
"immich_default": [
Endpoint("immich-server", 2283),
Endpoint("immich-database", 5432),
Endpoint("immich-redis", 6379),
Endpoint("immich-machine-learning", 8501),
],
"torrserv_proxy_default": [
Endpoint("fileserver", 8000),
Endpoint("reverseproxy", 8080),
],
"docker_socket_proxy_hermes_proxy": [
Endpoint("hermes-gateway", 8000),
],
"brencher_brencher-net": [
Endpoint("brencher-backend", 5000),
],
"brencher2_brencher-net": [
Endpoint("brencher-backend", 5000),
],
}
PUBLIC_SERVICES: List[Endpoint] = [
Endpoint("gitea.rudolf.keenetic.link", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("gitea.rudolf.keenetic.link", 80, proxy=SQUID_PROXY),
Endpoint("registry.rudolf.keenetic.link", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("registry.rudolf.keenetic.link", 5000, proxy=SQUID_PROXY),
Endpoint("deb.debian.org", 80, proxy=SQUID_PROXY),
Endpoint("deb.debian.org", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("github.com", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("githubusercontent.com", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("telegram.org", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("t.me", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("minimax.io", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("minimax.chat", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("minimax.com", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("pypi.org", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("google.com", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("cloudflare.com", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("wikipedia.org", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("mozilla.org", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("1.1.1.1", 443, proxy=SQUID_PROXY, tls=True),
Endpoint("8.8.8.8", 53),
Endpoint("8.8.4.4", 53),
]
GLOBAL_SERVICES: List[Endpoint] = sorted(
{ep for eps in NETWORK_MAP.values() for ep in eps} | set(PUBLIC_SERVICES),
key=lambda e: (e.name, e.port),
)
NEGATIVE: List[Endpoint] = [
Endpoint("169.254.169.254", 80),
Endpoint("metadata.google.internal", 80),
]
-36
View File
@@ -1,36 +0,0 @@
services:
connectivity-tester:
image: gitea.rudolf.keenetic.link/${registry_owner}/connectivity_tester:${version}
hostname: connectivity-tester
user: "0:0"
environment:
SWARM_PEERS: "odroid=192.168.1.96,rudolf-T=100.105.201.15"
volumes:
- /var/run/docker.sock:/var/run/docker.sock
- hermes_nfs_probe:/mnt/hermes-nfs-probe
healthcheck:
test: ["CMD-SHELL", "test -S /var/run/docker.sock"]
interval: 60s
timeout: 5s
retries: 3
start_period: 30s
deploy:
mode: global
resources:
limits:
memory: 128M
pids: 64
reservations:
memory: 64M
restart_policy:
condition: on-failure
delay: 10s
max_attempts: 3
volumes:
hermes_nfs_probe:
driver: local
driver_opts:
type: nfs
o: addr=192.168.1.96,rw,nfsvers=4
device: :/root/volumes/hermes/agent
-41
View File
@@ -1,41 +0,0 @@
import unittest
from host_checks import build_host_checks, parse_peers
class HostChecksTests(unittest.TestCase):
def test_parse_peers(self):
self.assertEqual(
parse_peers("odroid=192.168.1.96,rudolf-T=100.105.201.15"),
[("odroid", "192.168.1.96"), ("rudolf-T", "100.105.201.15")],
)
def test_parse_peers_ignores_invalid_entries(self):
self.assertEqual(
parse_peers("odroid=192.168.1.96,broken,rudolf-T="),
[("odroid", "192.168.1.96")],
)
def test_build_host_checks(self):
checks = build_host_checks(
[("odroid", "192.168.1.96"), ("rudolf-T", "100.105.201.15")],
[2377, 7946],
[7946, 4789],
)
self.assertEqual(
checks,
[
("odroid", "192.168.1.96", "tcp", 2377),
("odroid", "192.168.1.96", "tcp", 7946),
("odroid", "192.168.1.96", "udp", 7946),
("odroid", "192.168.1.96", "udp", 4789),
("rudolf-T", "100.105.201.15", "tcp", 2377),
("rudolf-T", "100.105.201.15", "tcp", 7946),
("rudolf-T", "100.105.201.15", "udp", 7946),
("rudolf-T", "100.105.201.15", "udp", 4789),
],
)
if __name__ == "__main__":
unittest.main()
-97
View File
@@ -1,97 +0,0 @@
# nginx config for the brencher-state-proxy sidecar.
#
# Path-locked reverse proxy for the brencher /state endpoint:
# - /state -> proxied to https://brencher.rudolf.keenetic.link/state
# - everything else -> 403 Forbidden
#
# This lets hermes-agent reach /state without going through squid's
# host-level allowlist (which can't see inside TLS) and without exposing
# the rest of brencher's surface area.
#
# Listen on 8080 (plain HTTP). No TLS -- this is a private internal
# API on the admin overlay, reachable only from inside the swarm. No
# host port is published.
#
# Upstream is the public URL on purpose -- the sidecar reaches the public
# brencher.rudolf.keenetic.link through the nginx-ingress overlay the same
# way any external client would, which means it doesn't need to be on the
# brencher container's private networks.
server {
listen 8080;
server_name _;
resolver 127.0.0.11 192.168.1.1 valid=30s ipv6=off;
resolver_timeout 5s;
# Tiny access log; default nginx format
access_log /var/log/nginx/brencher-state-access.log;
error_log /var/log/nginx/brencher-state-error.log warn;
# Healthcheck for swarm -- not /state, deliberately, so a 200 here
# doesn't accidentally imply /state is reachable from outside the
# allowlist.
location = /healthz {
access_log off;
return 200 "ok\n";
add_header Content-Type text/plain;
}
# The only allowed path. Exact match: query strings or trailing
# anything else fall through to the default 403.
location = /state {
set $brencher_upstream https://brencher.rudolf.keenetic.link;
proxy_pass $brencher_upstream/state;
proxy_http_version 1.1;
# Upstream is HTTPS; the sidecar trusts the LE-issued cert via
# the system CA bundle. SNI explicitly named so any future
# transparent-proxy middlebox doesn't break the handshake.
proxy_ssl_server_name on;
proxy_ssl_name brencher.rudolf.keenetic.link;
proxy_set_header Host brencher.rudolf.keenetic.link;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto https;
proxy_connect_timeout 5s;
proxy_read_timeout 30s;
proxy_send_timeout 10s;
# Don't cache -- state should be live.
proxy_buffering off;
add_header Cache-Control "no-store" always;
}
# Secondary brencher (runs on the docker host via host.docker.internal,
# reached as http://host.docker.internal:5002 -- same trick the brencher
# container itself uses in docker-compose.yml's extra_hosts). Only the
# /state2 path is exposed; everything else returns 403 like /state.
location = /state2 {
proxy_pass http://host.docker.internal:5002/state;
proxy_http_version 1.1;
# Cleartext HTTP upstream (port 5002), no TLS to wrangle.
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto https;
proxy_connect_timeout 5s;
proxy_read_timeout 30s;
proxy_send_timeout 10s;
proxy_buffering off;
add_header Cache-Control "no-store" always;
}
# Catch-all: deny everything else, including /state/ (with trailing
# slash), /state?foo=bar (with query), and any other path.
location / {
access_log off;
return 403 "forbidden: only /state is allowed through this proxy\n";
add_header Content-Type text/plain;
}
}
-2
View File
@@ -4,7 +4,6 @@ acl localnet src 192.168.0.0/16
acl allowed_domains dstdomain .minimax.io .minimax.chat .minimax.com .githubusercontent.com deb.debian.org gitea.rudolf.keenetic.link
acl telegram_domains dstdomain .telegram.org .t.me
acl stt_pkg_domains dstdomain .pypi.org .files.pythonhosted.org .huggingface.co .github.com .models.dev .registry.npmjs.org .cdn-lfs.huggingface.co .hf.co .xethub.hf.co .cas-server.xethub.hf.co
acl copilot_domains dstdomain api.github.com copilot-proxy.githubusercontent.com .githubcopilot.com copilot-telemetry.githubusercontent.com default.exp-tas.com marketplace.visualstudio.com vscode.blob.core.windows.net .githubassets.com .vscode-unpkg.net
acl telegram_ips dst 91.108.4.0/22 91.108.8.0/22 91.108.12.0/22 91.108.16.0/22 91.108.20.0/22 91.108.56.0/22 149.154.160.0/20 185.76.151.0/24
acl CONNECT method CONNECT
acl SSL_ports port 443
@@ -15,7 +14,6 @@ http_access deny CONNECT !SSL_ports
http_access allow localnet allowed_domains
http_access allow localnet telegram_domains
http_access allow localnet stt_pkg_domains
http_access allow localnet copilot_domains
http_access allow localnet telegram_ips
http_access deny all
+11 -49
View File
@@ -64,10 +64,10 @@ services:
command: gateway run
user: "0:0"
networks:
# No admin overlay membership here: the sidecars (brencher-state-proxy,
# docker_socket_proxy) are on hermes_proxy too, so DNS resolves
# without expanding hermes-gateway's L3 footprint. The NO_PROXY
# list below ensures those requests skip squid.
# No admin overlay membership here: the docker_socket_proxy sidecar
# is on hermes_proxy too, so DNS resolves without expanding
# hermes-gateway's L3 footprint. The NO_PROXY list below ensures
# those requests skip squid.
- hermes_proxy
volumes:
- hermes_gateway_data:/opt/data
@@ -84,21 +84,21 @@ services:
HTTPS_PROXY: http://squid:3128
http_proxy: http://squid:3128
https_proxy: http://squid:3128
# The sidecars added in this PR (brencher-state-proxy, docker_socket_proxy)
# are reachable on the hermes_proxy overlay. They are NOT public services, so
# traffic to them MUST NOT be routed through squid (which can't reach
# them anyway and would 403 the request).
# The docker_socket_proxy sidecar is reachable on the hermes_proxy
# overlay. It is NOT a public service, so traffic to it MUST NOT be
# routed through squid (which can't reach it anyway and would 403
# the request).
# Comma-separated list. Format accepted by curl/requests/urllib:
# bare hostnames, IPs, CIDR ranges. Order doesn't matter; case-insensitive.
# 127.0.0.1 - loopback
# .local - mDNS / link-local (covers host.docker.internal)
# squid, hermes-gateway, hermes-agent - same-overlay services that
# hermes-gateway itself needs to talk to without going through squid
# brencher-state-proxy, docker_socket_proxy - the new sidecars
# docker_socket_proxy - the sidecar
# 10.0.0.0/8 - the swarm's typical overlay subnet (defensive: covers
# any IP-direct call that bypasses DNS)
NO_PROXY: localhost,127.0.0.1,.local,squid,hermes-gateway,hermes-agent,brencher-state-proxy,docker_socket_proxy,10.0.0.0/8
no_proxy: localhost,127.0.0.1,.local,squid,hermes-gateway,hermes-agent,brencher-state-proxy,docker_socket_proxy,10.0.0.0/8
NO_PROXY: localhost,127.0.0.1,.local,squid,hermes-gateway,hermes-agent,docker_socket_proxy,10.0.0.0/8
no_proxy: localhost,127.0.0.1,.local,squid,hermes-gateway,hermes-agent,docker_socket_proxy,10.0.0.0/8
depends_on:
- squid
deploy:
@@ -193,42 +193,6 @@ services:
delay: 5s
max_attempts: 3
brencher-state-proxy:
# Path-locked reverse proxy for brencher.rudolf.keenetic.link/state.
# Lets hermes-agent reach /state without going through squid (which
# can't see inside TLS) and without exposing the rest of brencher.
# See hermes_squid/brencher-state-proxy.conf for the path rules.
image: nginx:alpine
hostname: brencher-state-proxy
extra_hosts:
# Lets the sidecar reach the secondary brencher at
# http://host.docker.internal:5002/state -- same trick the brencher
# container itself uses in its own docker-compose.yml.
- "host.docker.internal:host-gateway"
networks:
# hermes_proxy: same as docker_socket_proxy. hermes-gateway is
# already on this overlay, so DNS works. Egress is allowed despite
# the `internal: true` flag (which only blocks external ingress,
# not outbound traffic). The sidecar's upstream
# proxy_pass https://brencher.rudolf.keenetic.link/state is a
# public-internet HTTPS call, not a service-name lookup, so
# nginx_ingress membership is not required.
- hermes_proxy
configs:
- source: brencher_state_proxy_conf
target: /etc/nginx/conf.d/brencher-state-proxy.conf
deploy:
mode: replicated
replicas: 1
placement:
constraints:
- node.hostname == odroid # must run where the secret is mounted (and where the public ingress routes back to)
restart_policy:
condition: on-failure
delay: 5s
max_attempts: 3
networks:
hermes_proxy:
driver: overlay
@@ -247,8 +211,6 @@ configs:
file: ./squid.conf
nginx_hermes_conf:
file: ./nginx-hermes.conf
brencher_state_proxy_conf:
file: ./brencher-state-proxy.conf
volumes:
squid_etc:
-29
View File
@@ -1,29 +0,0 @@
# Minimal image that downloads AWL binary from GitHub releases
FROM alpine:latest
# Install dependencies for downloading and running AWL
RUN apk add --no-cache ca-certificates curl
# Download AWL binary for Linux amd64 from v0.17.0 release
# Extract from the tar.gz archive
RUN set -e; \
mkdir -p /tmp/awl-download; \
cd /tmp/awl-download; \
curl -L "https://github.com/anywherelan/awl/releases/download/v0.17.0/awl-linux-amd64-v0.17.0.tar.gz" -o awl.tar.gz; \
tar -xzf awl.tar.gz; \
mv awl /usr/local/bin/awl; \
chmod +x /usr/local/bin/awl; \
rm -rf /tmp/awl-download
# Copy entrypoint script for automatic key generation
COPY entrypoint.sh /usr/local/bin/entrypoint.sh
RUN chmod +x /usr/local/bin/entrypoint.sh
# Create directory for storing keys and configuration
WORKDIR /data
# Expose ports: 8639 for Web UI, 4363 for P2P network
EXPOSE 8639 4363
# Run the entrypoint script that starts the AWL node
ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
-147
View File
@@ -1,147 +0,0 @@
### Start the Node
The node will automatically generate its encryption key and configuration on first startup:
```bash
docker run -d \
--name anywherelan-node \
--restart unless-stopped \
--privileged \
-v awl-config:/data \
anywherelan:v0.17.0
```
**Flags explained:**
- `-d`: Run in background (detached mode)
- `--restart unless-stopped`: Auto-restart on failure
- `--privileged`: Required for network interface creation (TUN/TAP device)
- `-v awl-config:/data`: Create and mount a named volume for config/key storage
The container will generate `network.key` and `config.json` in the `awl-config` volume on first run.
### Verify Node is Running
```bash
docker ps | grep anywherelan
docker logs anywherelan-node
```
## Web Interface
The node exposes a web interface on port 8639. Access it at:
```
http://localhost:8639
```
## Docker Compose Alternative
Create a `docker-compose.yml` file:
```yaml
version: '3.8'
services:
anywherelan:
image: anywherelan:v0.17.0
container_name: anywherelan-node
restart: unless-stopped
privileged: true
volumes:
- awl-config:/data
ports:
- "8639:8639"
volumes:
awl-config:
```
Then start with:
```bash
docker-compose up -d
```
The network key and configuration will be automatically generated on first startup.
## Stopping the Node
```bash
docker stop anywherelan-node
```
## Removing the Node
```bash
docker rm anywherelan-node
```
To also remove the configuration volume:
```bash
docker volume rm awl-config
```
⚠️ **Warning**: Removing the volume will delete all configuration and keys. Only do this if you want a fresh start.
## Configuration
Configuration is stored in the `awl-config` volume:
- `network.key`: Your node's encryption key (auto-generated)
- `config.json`: Node configuration (auto-generated)
To access or backup configuration:
```bash
# View volume location
docker volume inspect awl-config
# Backup configuration
docker run --rm -v awl-config:/data -v $(pwd):/backup alpine cp -r /data /backup/awl-config-backup
# Restore configuration
docker run --rm -v awl-config:/data -v $(pwd):/backup alpine cp -r /backup/awl-config-backup /data
```
## Troubleshooting
### Permission Denied Error
The `--privileged` flag is required for creating virtual network interfaces (TUN/TAP devices).
### Port Already in Use
If port 8639 is taken, map to a different port:
```bash
docker run -d \
--name anywherelan-node \
--restart unless-stopped \
--privileged \
-p 8640:8639 \
-v awl-config:/data \
anywherelan:v0.17.0
```
Then access the web interface at `http://localhost:8640`
### View Logs
```bash
docker logs -f anywherelan-node
```
### Missing TUN Device
If you see "create tun: CreateTUN failed", ensure:
1. Docker is running with `--privileged` flag
2. The host has `/dev/net/tun` (on Linux hosts)
3. On non-Linux systems (Mac/Windows), Docker Desktop must have proper permissions
### Inspect Volume
To see where Docker stores your volume:
```bash
docker volume inspect awl-config
```
## Version
Built from tag: `v0.17.0`
## References
- [Anywherelan Official Repository](https://github.com/anywherelan/awl)
- [Project Documentation](https://github.com/anywherelan/awl/blob/main/README.md)
-29
View File
@@ -1,29 +0,0 @@
version: '3.8'
services:
anywherelan:
image: anywherelan:v0.17.0
container_name: anywherelan-node
restart: unless-stopped
privileged: true
volumes:
- awl-config:/data
ports:
- "8639:8639"
networks:
- anywherelan-net
environment:
- TZ=UTC
logging:
driver: "json-file"
options:
max-size: "100m"
max-file: "10"
volumes:
awl-config:
driver: local
networks:
anywherelan-net:
driver: bridge
-13
View File
@@ -1,13 +0,0 @@
#!/bin/sh
set -e
# Ensure /data directory exists
mkdir -p /data
# Symlink the config directory to home config location so AWL can find it
mkdir -p /root/.config/anywherelan
ln -sf /data/config_awl.json /root/.config/anywherelan/config_awl.json 2>/dev/null || true
# Run the AWL node (no command needed, it starts the server by default)
exec awl
-62
View File
@@ -1,62 +0,0 @@
services:
beszel:
image: henrygd/beszel:latest
environment:
APP_URL: http://localhost:8090
ports:
- 8090:8090
volumes:
- beszel_data:/beszel_data
deploy:
mode: replicated
replicas: 1
placement:
constraints:
- node.role == manager
restart_policy:
condition: on-failure
# healthcheck:
# test: ['CMD', '/beszel', 'health', '--url', 'http://localhost:8090']
# interval: 120s
# start_period: 10s
# timeout: 5s
beszel-agent:
image: henrygd/beszel-agent:latest
container_name: beszel-agent
restart: unless-stopped
network_mode: host
volumes:
- beszel_agent_data:/var/lib/beszel-agent
- beszel_socket:/beszel_socket
- /var/run/docker.sock:/var/run/docker.sock:ro
environment:
LISTEN: /beszel_socket/beszel.sock
HUB_URL: http://100.105.201.15:8090
KEY: 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOKMyuYnUwzLlYS7yZxcs1psBzRl7n9MhoZQ0Kek3XtV'
TOKEN: a2c7cabb-c849-4f64-89cf-1cf9afbfb018
# healthcheck:
# test: ['CMD', '/agent', 'health']
# interval: 120s
beszel-agent1:
image: henrygd/beszel-agent
container_name: beszel-agent1
restart: unless-stopped
network_mode: host
volumes:
- /var/run/docker.sock:/var/run/docker.sock:ro
- beszel_agent_data:/var/lib/beszel-agent
# monitor other disks / partitions by mounting a folder in /extra-filesystems
# - /mnt/disk/.beszel:/extra-filesystems/sda1:ro
environment:
LISTEN: 45876
KEY: 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOKMyuYnUwzLlYS7yZxcs1psBzRl7n9MhoZQ0Kek3XtV'
TOKEN: cd057ffc-8af3-414f-bbd4-3d0c9986b8ba
HUB_URL: http://100.105.201.15:8090
volumes:
beszel_agent_data:
volumes:
beszel_data:
beszel_agent_data:
beszel_socket:
-39
View File
@@ -1,39 +0,0 @@
#!/usr/bin/env bash
# Custom entrypoint for the act_runner cache-server.
#
# The upstream image's /usr/local/bin/run.sh hardcodes `act_runner daemon`
# and ignores argv. So we replace the entrypoint here.
#
# cache-server also requires cache.external_secret to be set in config.yaml
# as a literal value (not a file path). We keep the value in a Docker
# secret and substitute it into a runtime copy of config.yaml before
# exec'ing act_runner.
set -eu
SECRET_FILE="${ACT_RUNNER_EXTERNAL_SECRET_FILE:-/run/secrets/gitea_cache_secret}"
CONFIG_FILE="${CONFIG_FILE:-/cfg/config.yaml}"
if [[ ! -s "$SECRET_FILE" ]]; then
echo "cache-server-run: secret file $SECRET_FILE is missing or empty" >&2
exit 1
fi
if [[ ! -f "$CONFIG_FILE" ]]; then
echo "cache-server-run: config file $CONFIG_FILE not found" >&2
exit 1
fi
SECRET_VALUE="$(cat "$SECRET_FILE")"
# Render a copy of config.yaml with cache.external_secret replaced by the
# secret's literal value. /cfg/config.yaml is a read-only Swarm config mount,
# so we cannot edit it in place. Use a plain-file replacement — the
# upstream config.yaml has the form `external_secret: ""` (or any literal)
# on a single line, so a simple sed is safe.
RUNTIME_CONFIG="/tmp/config.runtime.yaml"
cp "$CONFIG_FILE" "$RUNTIME_CONFIG"
# Replace the value after `external_secret:` on a single line.
# Use a delimiter unlikely to appear in the secret.
sed -i "s|^ external_secret:.*$| external_secret: \"${SECRET_VALUE}\"|" "$RUNTIME_CONFIG"
exec /usr/local/bin/act_runner cache-server --config "$RUNTIME_CONFIG"
+18 -8
View File
@@ -70,16 +70,26 @@ runner:
- "ubuntu-22.04:docker://docker.gitea.com/runner-images:ubuntu-22.04"
cache:
# Enable cache server to use actions/cache.
enabled: true
# The directory to store the cache data.
# If it's empty, the cache data will be stored in $HOME/.cache/actcache.
dir: ""
# Point at the dedicated gitea_cache_server container on the `gitea` overlay.
# The runner's built-in cache server binds to an interface unreachable from
# per-job container networks, causing ~9 min/job wasted on
# "reserveCache failed: connect ETIMEDOUT" (see gitea/actions/runs/16 job #248).
host: "gitea_cache_server"
port: 37051
external_server: "http://gitea_cache_server:37051/"
# Read from the gitea_cache_secret Docker secret (see stack-compose.yml).
# The host of the cache server.
# It's not for the address to listen, but the address to connect from job containers.
# So 0.0.0.0 is a bad choice, leave it empty to detect automatically.
host: ""
# The port of the cache server.
# 0 means to use a random available port.
port: 0
# The external cache server URL. Valid only when enable is true.
# If it's specified, act_runner will use this URL as the ACTIONS_CACHE_URL rather than start a server by itself.
# The URL should generally end with "/".
# Requires external_secret below to be set to the same value on both this runner and the cache-server.
external_server: ""
# Shared secret between this runner and the external `act_runner cache-server`. Required when external_server
# (or `act_runner cache-server`) is in use: the runner pre-registers each job's ACTIONS_RUNTIME_TOKEN with the
# cache-server, and the cache-server enforces bearer auth + per-repo cache isolation.
external_secret: ""
container:
-36
View File
@@ -1,36 +0,0 @@
#!/usr/bin/env bash
# Custom entrypoint for the gitea_runner service.
#
# Same trick as cache-server-run.sh: act_runner requires cache.external_secret
# to be a literal string in config.yaml, not a file path. We keep the value in
# a Docker secret and substitute it into a runtime copy of config.yaml, then
# exec the image's own /usr/local/bin/run.sh (which handles registration +
# daemon).
set -eu
SECRET_FILE="${ACT_RUNNER_EXTERNAL_SECRET_FILE:-/run/secrets/gitea_cache_secret}"
CONFIG_FILE="${CONFIG_FILE:-/cfg/config.yaml}"
if [[ ! -s "$SECRET_FILE" ]]; then
echo "runner-run: secret file $SECRET_FILE is missing or empty" >&2
exit 1
fi
if [[ ! -f "$CONFIG_FILE" ]]; then
echo "runner-run: config file $CONFIG_FILE not found" >&2
exit 1
fi
SECRET_VALUE="$(cat "$SECRET_FILE")"
# Render a runtime copy with the secret substituted in.
RUNTIME_CONFIG="/tmp/config.runtime.yaml"
cp "$CONFIG_FILE" "$RUNTIME_CONFIG"
sed -i "s|^ external_secret:.*$| external_secret: \"${SECRET_VALUE}\"|" "$RUNTIME_CONFIG"
# The upstream run.sh reads $CONFIG_FILE from the environment to pass
# --config to act_runner. Override it so registration + daemon both pick up
# the runtime copy with the substituted secret.
export CONFIG_FILE="$RUNTIME_CONFIG"
exec /usr/local/bin/run.sh
+1 -55
View File
@@ -1,8 +1,4 @@
# Gitea Actions stack. Uses a dedicated act_runner cache-server so that
# job-container networks can reach the cache backend via Swarm service DNS
# (gitea_cache_server:37051) instead of the unreachable IP the built-in
# cache server binds to per-job.
services:
server:
image: docker.gitea.com/gitea:1.27.0
@@ -23,7 +19,6 @@ services:
runner:
image: docker.io/gitea/act_runner:nightly
entrypoint: ["/usr/local/bin/runner-run.sh"]
environment:
CONFIG_FILE: /cfg/config.yaml
GITEA_INSTANCE_URL: "https://gitea.rudolf.keenetic.link"
@@ -31,51 +26,14 @@ services:
GITEA_RUNNER_NAME: "runner1"
GITEA_RUNNER_LABELS: "runner_label1"
# GITEA_RUNNER_EPHEMERAL: "1"
ACT_RUNNER_EXTERNAL_SECRET_FILE: /run/secrets/gitea_cache_secret
networks:
- gitea
# - nginx_ingress
configs:
- source: runner_config
target: /cfg/config.yaml
mode: 0444
- source: runner_run
target: /usr/local/bin/runner-run.sh
mode: 0755
volumes:
- /var/run/docker.sock:/var/run/docker.sock
secrets:
- gitea_cache_secret
deploy:
restart_policy:
condition: on-failure
delay: 5s
max_attempts: 3
cache_server:
# The image's /usr/local/bin/run.sh hardcodes `act_runner daemon` and
# ignores argv, so passing `cache-server` via `command` doesn't help —
# the daemon path is always taken. We replace the entrypoint by
# mounting our own script via a Swarm config (file mount), then have it
# substitute the Docker secret literal into a runtime copy of
# config.yaml's cache.external_secret and exec act_runner.
image: docker.io/gitea/act_runner:nightly
entrypoint: ["/usr/local/bin/cache-server-run.sh"]
environment:
ACT_RUNNER_EXTERNAL_SECRET_FILE: /run/secrets/gitea_cache_secret
networks:
- gitea
volumes:
- cache_data:/root/.cache/actcache
secrets:
- gitea_cache_secret
configs:
- source: runner_config
target: /cfg/config.yaml
mode: 0444
- source: cache_server_run
target: /usr/local/bin/cache-server-run.sh
mode: 0755
networks:
nginx_ingress:
@@ -86,17 +44,6 @@ networks:
configs:
runner_config:
file: ./config.yaml
cache_server_run:
file: ./cache-server-run.sh
runner_run:
file: ./runner-run.sh
secrets:
gitea_cache_secret:
# Create once before first deploy (or to rotate):
# echo -n "$SECRET" | docker secret create gitea_cache_secret -
# Then `docker stack deploy -c stack-compose.yml gitea`.
external: true
volumes:
data:
@@ -105,5 +52,4 @@ volumes:
type: nfs
o: addr=192.168.1.96
device: :/root/volumes/gitea
# device: :/media/DATA/runtime/gitea
cache_data:
# device: :/media/DATA/runtime/gitea
-10
View File
@@ -51,15 +51,5 @@ networks:
volumes:
hermes_gateway_data:
driver: local
driver_opts:
type: nfs
o: addr=192.168.1.96
device: :/root/volumes/hermes/gateway
hermes_agent_data:
driver: local
driver_opts:
type: nfs
o: addr=192.168.1.96
device: :/root/volumes/hermes/agent
-11
View File
@@ -1,11 +0,0 @@
FROM ubuntu:20.04
RUN apt-get update && apt-get install -y curl && rm -rf /var/lib/apt/lists/*
RUN curl -fsSL https://pkgs.netbird.io/install.sh | sh
RUN mkdir -p /scripts && \
printf '#!/bin/bash\necho "=== Starting NetBird Entrypoint ==="\necho "Current user: $(whoami)"\necho "PATH: $PATH"\necho "Checking for netbird..."\nwhich netbird || echo "netbird not found in PATH"\necho "Attempting to start NetBird..."\nif command -v netbird &> /dev/null; then\n netbird up -F &\n NETBIRD_PID=$!\n echo "NetBird started with PID $NETBIRD_PID"\n trap "kill $NETBIRD_PID" TERM INT\n wait $NETBIRD_PID\nelse\n echo "ERROR: netbird command not available"\n echo "=== Container will keep running for debugging ==="\n tail -f /dev/null\nfi\n' > /scripts/entrypoint.sh && \
chmod +x /scripts/entrypoint.sh
ENTRYPOINT ["/scripts/entrypoint.sh"]
-117
View File
@@ -1,117 +0,0 @@
# Netbird Docker Deployment
Docker Compose configurations and automated setup for Netbird - a zero-trust mesh VPN solution.
## Quick Start
**Start here**: Run the automated setup script
```bash
cd /media/DATA/src/ansible/poc/netbird
./setup-netbird.sh
```
Or manually clone official repo:
```bash
git clone https://github.com/netbirdio/netbird.git
cd netbird/infrastructure_files/compose
docker-compose up -d
```
## ⚠️ Important: OIDC Bug in Pre-built Images
Pre-built Netbird Docker images crash on startup with segmentation fault in OIDC code. **Do not use local stack-server.yml directly.** Use official repository or setup script instead.
## Files
| File | Purpose |
|------|---------|
| **setup-netbird.sh** | ⭐ Automated setup script |
| **SETUP_NETBIRD_OFFICIAL.md** | Official deployment methods |
| **stack-server.yml** | Local Docker Compose (⚠️ management crashes) |
| **stack-client.yml** | Client instance |
| **VERSIONS.md** | Version reference |
## Client Deployment
Once you have working Netbird management:
```bash
export MANAGEMENT_URL="https://your-server:33073"
export SETUP_KEY="from-dashboard"
docker-compose -f stack-client.yml up -d
```
Multiple clients:
```bash
docker-compose -f stack-client.yml -p client1 up -d
docker-compose -f stack-client.yml -p client2 up -d
```
## Android Setup (Same WiFi)
1. Get server IP: `hostname -I`
2. Open firewall: `sudo ufw allow 33073`
3. Android app: Enter `http://<server-ip>:33073` + setup key
Test: `curl http://<ip>:33073/api/health`
## Docker Commands
```bash
# Local infrastructure only (management will crash)
docker-compose -f stack-server.yml up -d
docker-compose -f stack-server.yml ps
docker-compose -f stack-server.yml logs -f
# Stop
docker-compose -f stack-server.yml down
# Database access
docker-compose -f stack-server.yml exec postgres \
psql -U netbird -d netbird
# Client commands
docker-compose -f stack-client.yml logs -f
```
## Services
| Service | Port |
|---------|------|
| Management API | 33073 |
| Signal | 10000 |
| Dashboard | 80 |
| PostgreSQL | 5432 |
| Redis | 6379 |
| TURN | 3478, 5349 |
## Environment Variables
```bash
# Server (set before docker-compose up)
POSTGRES_PASSWORD=netbird123
NETBIRD_MGMT_SINGLE_ACCOUNT_MODE=true
NETBIRD_MGMT_LOG_LEVEL=info
# Client
MANAGEMENT_URL=https://server:33073
SETUP_KEY=your-key
```
## Requirements
**Server**: Docker, 2GB RAM, ports 80, 33073, 10000, 3478, 5349
**Client**: Docker with /dev/net/tun access, NET_ADMIN capability
## Troubleshooting
**Management crashes**: Use official repo or setup script
**Can't connect from Android**: Check firewall, use correct IP (not localhost)
**Database issues**: Check logs with `docker-compose logs postgres`
## Resources
- [Official Docs](https://docs.netbird.io/)
- [GitHub](https://github.com/netbirdio/netbird)
- Detailed guide in [SETUP_NETBIRD_OFFICIAL.md](SETUP_NETBIRD_OFFICIAL.md)
-51
View File
@@ -1,51 +0,0 @@
{
"Datadir": "/var/lib/netbird/",
"DataStoreEncryptionKey": "8zd4m/LHfTvaQtst9ZW29BpZ0U1/RrvRvhtE1y7OdKg=",
"StoreConfig": {
"Engine": "postgres"
},
"HttpConfig": {
"Address": "0.0.0.0",
"Port": 33073,
"CertFile": "",
"CertKey": ""
},
"Stuns": [
{
"Proto": "udp",
"URI": "stun:3468",
"Username": "",
"Password": ""
}
],
"TURNConfig": {
"Turns": [
{
"Proto": "udp",
"URI": "turn:3478",
"Username": "",
"Password": ""
}
],
"CredentialsMap": {}
},
"Signal": {
"URI": "signal:10000",
"Proto": "grpc"
},
"Relay": {
"URI": "relay:7835"
},
"ReverseProxy": {
"TrustedHTTPProxies": null,
"TrustedHTTPProxiesCount": 0,
"TrustedPeers": null,
"AccessLogRetentionDays": 0,
"AccessLogCleanupIntervalHours": 0
},
"IdpManagerConfig": null,
"DeviceAuthorizationFlow": null,
"PKCEAuthorizationFlow": null,
"DisableDefaultPolicy": false,
"EmbeddedIdP": null
}
-33
View File
@@ -1,33 +0,0 @@
version: '3.8'
services:
netbird-client:
image: netbirdio/client:latest
container_name: netbird-client
cap_add:
- NET_ADMIN
devices:
- /dev/net/tun
environment:
NETBIRD_MANAGEMENT_URL: ${MANAGEMENT_URL:-https://management:33073}
NETBIRD_LOG_LEVEL: info
# Use setup key for automatic enrollment (optional)
NETBIRD_SETUP_KEY: ${SETUP_KEY:-}
volumes:
- netbird_client_data:/etc/netbird
- /etc/resolv.conf:/etc/resolv.conf:ro
ports:
- "41641:41641/udp"
networks:
- netbird
restart: unless-stopped
dns:
- 8.8.8.8
- 8.8.4.4
volumes:
netbird_client_data:
networks:
netbird:
driver: bridge
-118
View File
@@ -1,118 +0,0 @@
version: '3.8'
services:
postgres:
image: postgres:15-alpine
container_name: netbird-postgres
environment:
POSTGRES_DB: netbird
POSTGRES_USER: netbird
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-netbird123}
volumes:
- postgres_data:/var/lib/postgresql/data
ports:
- "5432:5432"
healthcheck:
test: ["CMD-SHELL", "pg_isready -U netbird"]
interval: 10s
timeout: 5s
retries: 5
redis:
image: redis:7-alpine
container_name: netbird-redis
ports:
- "6379:6379"
volumes:
- redis_data:/data
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 10s
timeout: 5s
retries: 5
management:
image: netbirdio/management:0.70.5
container_name: netbird-management
depends_on:
postgres:
condition: service_healthy
redis:
condition: service_healthy
environment:
# PostgreSQL configuration
NETBIRD_MGMT_DATASTORE: postgres
NETBIRD_MGMT_DATASTORE_HOST: postgres
NETBIRD_MGMT_DATASTORE_PORT: "5432"
NETBIRD_MGMT_DATASTORE_DB: netbird
NETBIRD_MGMT_DATASTORE_USER: netbird
NETBIRD_MGMT_DATASTORE_PASSWORD: ${POSTGRES_PASSWORD:-netbird123}
# Redis
NETBIRD_MGMT_DATASTORE_CACHE: "redis"
NETBIRD_MGMT_DATASTORE_REDIS_URL: "redis://redis:6379"
# Management API
NETBIRD_MGMT_API_PORT: "33073"
NETBIRD_MGMT_LOG_LEVEL: debug
NETBIRD_MGMT_SINGLE_ACCOUNT_MODE: "true"
NETBIRD_MGMT_DISABLE_SINGLE_ACCOUNT_MODE_REQUIRE_CUSTOM_DOMAIN: "true"
# OIDC disabled
NETBIRD_MGMT_AUTH_OIDC_CONFIGURATION_ENDPOINT: ""
ports:
- "33073:33073"
volumes:
- management_data:/var/lib/netbird
- ./management.json:/etc/netbird/management.json
networks:
- netbird
restart: "no"
signal:
image: netbirdio/signal:0.70.5
container_name: netbird-signal
environment:
NETBIRD_SIGNAL_LOG_LEVEL: info
NETBIRD_SIGNAL_PORT: 10000
ports:
- "10000:10000/tcp"
- "10000:10000/udp"
networks:
- netbird
depends_on:
- management
dashboard:
image: netbirdio/dashboard:latest
container_name: netbird-dashboard
environment:
REACT_APP_API_URL: ${DASHBOARD_API_URL:-http://localhost:33073}
ports:
- "80:80"
networks:
- netbird
depends_on:
- management
coturn:
image: coturn/coturn:latest
container_name: netbird-turn
ports:
- "3478:3478/tcp"
- "3478:3478/udp"
- "5349:5349/tcp"
- "5349:5349/udp"
volumes:
- ./coturn.conf:/etc/coturn/turnserver.conf:ro
- turn_data:/var/lib/coturn
networks:
- netbird
restart: unless-stopped
volumes:
postgres_data:
redis_data:
management_data:
turn_data:
networks:
netbird:
driver: bridge
-21
View File
@@ -1,21 +0,0 @@
Code-Server (code-server) Docker Swarm stack
Deploy a lightweight web-accessible VS Code (code-server) on Docker Swarm.
Quick usage
- Edit `stack-code-server.yml` to set a secure `PASSWORD` (or use Docker secrets).
- Deploy the stack:
```bash
docker stack deploy -c stack-code-server.yml code-server
```
Access
- Open http://<HOST>:8443 or https depending on the image configuration. Log in with the password set in the compose file.
Notes
- The service stores config in a named volume `code_server_data` and mounts `./workspace` into the container at `/config/workspace`.
- For production, replace the `PASSWORD` with a Docker secret and consider putting the service behind a reverse proxy/traefik with TLS.
-22
View File
@@ -1,22 +0,0 @@
version: '3.8'
services:
code-server:
image: linuxserver/code-server:latest
ports:
- "8443:8443"
environment:
PUID: 1000
PGID: 1000
TZ: UTC
PASSWORD: changeme
volumes:
- config:/config
- workspace:/config/workspace
deploy:
replicas: 1
restart_policy:
condition: on-failure
volumes:
config:
workspace: