forked from admin/ansible
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8b4695733b | ||
|
|
c8e33e2600 | ||
|
|
0a9e795b8f | ||
|
|
db6a79e1d0 | ||
|
|
7e69577609 | ||
|
|
e9de263ee1 | ||
|
|
1de62b3385 | ||
|
|
74c3b435e8 | ||
|
|
82487b1565 | ||
|
|
e3f195f4bc | ||
|
|
9436806f78 | ||
|
|
ff08677d12 | ||
|
|
0998f4bc05 | ||
|
|
f071c7fb43 | ||
|
|
c3b7128006 | ||
|
|
0c95d8e1ee |
@@ -2,16 +2,12 @@ name: release
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- 'auto-*'
|
||||
tags:
|
||||
- 'v*'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
tag:
|
||||
description: 'Image tag (e.g. auto-<sha>)'
|
||||
required: false
|
||||
default: latest
|
||||
required: true
|
||||
type: string
|
||||
|
||||
jobs:
|
||||
@@ -28,15 +24,8 @@ jobs:
|
||||
username: ${{ secrets.REGISTRY_USERNAME }}
|
||||
password: ${{ secrets.REGISTRY_TOKEN }}
|
||||
|
||||
- name: resolve version
|
||||
id: version
|
||||
run: |
|
||||
if [ -n "${{ inputs.tag }}" ] && [ "${{ inputs.tag }}" != "latest" ]; then
|
||||
echo "VERSION=${{ inputs.tag }}" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "VERSION=${GITHUB_REF_NAME##*/}" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
echo "VERSION=$VERSION"
|
||||
- name: setup buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: build and push
|
||||
uses: docker/build-push-action@v6
|
||||
@@ -44,6 +33,7 @@ jobs:
|
||||
context: connectivity_tester
|
||||
file: connectivity_tester/Dockerfile
|
||||
push: true
|
||||
platforms: linux/amd64,linux/arm64,linux/arm/v7
|
||||
tags: |
|
||||
gitea.rudolf.keenetic.link/hermes/connectivity_tester:${{ steps.version.outputs.VERSION }}
|
||||
gitea.rudolf.keenetic.link/hermes/connectivity_tester:latest
|
||||
gitea.rudolf.keenetic.link/admin/connectivity_tester:${{ inputs.tag || github.ref_name }}
|
||||
gitea.rudolf.keenetic.link/admin/connectivity_tester:latest
|
||||
|
||||
@@ -4,7 +4,9 @@
|
||||
docker swarm init --advertise-addr 192.168.1.96 --force-new-cluster
|
||||
Swarm initialized: current node (j9bljgaqh9qch2nmozvu2bprs) is now a manager.
|
||||
To add a worker to this swarm, run the following command:
|
||||
docker swarm join --token SWMTKN-1-0eb9ypg3ouig0ac493scmjtogida7ddy79xr2e9kkvaitoz0y0-7t5lg5sx8q20uqz2uandsblc9 192.168.1.96:2377
|
||||
|
||||
docker swarm join --token <token> \
|
||||
--advertise-addr 100.105.201.15 192.168.1.96:2377
|
||||
|
||||
## Prod instances
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM python:3.12-slim
|
||||
FROM python:3.12-alpine
|
||||
|
||||
WORKDIR /app
|
||||
COPY . /app/
|
||||
|
||||
@@ -13,9 +13,11 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import ssl
|
||||
import socket
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Iterable, List, Set, Tuple
|
||||
|
||||
import docker
|
||||
@@ -23,11 +25,17 @@ from docker.models.containers import Container
|
||||
from docker.models.networks import Network
|
||||
|
||||
from network_map import Endpoint, GLOBAL_SERVICES, NEGATIVE, NETWORK_MAP, PUBLIC_SERVICES
|
||||
from host_checks import build_host_checks, parse_peers, tcp_probe, udp_probe
|
||||
|
||||
LOG = logging.getLogger("connectivity-tester")
|
||||
|
||||
FALLBACK_TCP_TIMEOUT_S = 3.0
|
||||
HTTP_CONNECT_TIMEOUT_S = 5.0
|
||||
HOST_TCP_PORTS = (2377, 7946, 2049)
|
||||
HOST_UDP_PORTS = (7946, 4789)
|
||||
DIRECT_INTERNET_HOST = "api.telegram.org"
|
||||
DIRECT_INTERNET_PORT = 443
|
||||
NFS_PROBE_PATH = os.environ.get("NFS_PROBE_PATH", "/mnt/hermes-nfs-probe")
|
||||
|
||||
|
||||
def _socket_path() -> str:
|
||||
@@ -99,6 +107,25 @@ async def _tcp_probe(host: str, port: int, timeout: float) -> Tuple[bool, str]:
|
||||
return False, f"{type(exc).__name__}: {exc}"
|
||||
|
||||
|
||||
async def _tls_probe(host: str, port: int, timeout: float) -> Tuple[bool, str]:
|
||||
context = ssl.create_default_context()
|
||||
try:
|
||||
_, writer = await asyncio.wait_for(
|
||||
asyncio.open_connection(host, port, ssl=context, server_hostname=host),
|
||||
timeout=timeout,
|
||||
)
|
||||
writer.close()
|
||||
try:
|
||||
await writer.wait_closed()
|
||||
except Exception:
|
||||
pass
|
||||
return True, "tls handshake ok"
|
||||
except asyncio.TimeoutError:
|
||||
return False, "timeout"
|
||||
except (OSError, ssl.SSLError) as exc:
|
||||
return False, f"{type(exc).__name__}: {exc}"
|
||||
|
||||
|
||||
async def _proxy_connect(endpoint: Endpoint, timeout: float) -> Tuple[bool, str]:
|
||||
if endpoint.proxy is None:
|
||||
return await _tcp_probe(endpoint.name, endpoint.port, timeout=timeout)
|
||||
@@ -214,6 +241,59 @@ async def _check_negative(endpoints: List[Endpoint],
|
||||
return leaks
|
||||
|
||||
|
||||
async def _check_host_connectivity(
|
||||
peers: List[Tuple[str, str]], timeout: float
|
||||
) -> None:
|
||||
checks = build_host_checks(peers, HOST_TCP_PORTS, HOST_UDP_PORTS)
|
||||
LOG.info("--- host connectivity checks (%d probes) ---", len(checks))
|
||||
for name, address, protocol, port in checks:
|
||||
if protocol == "tcp":
|
||||
ok, detail = await tcp_probe(address, port, timeout)
|
||||
else:
|
||||
ok, detail = await udp_probe(address, port, timeout)
|
||||
label = f"{name} {address}:{port}/{protocol}"
|
||||
if ok:
|
||||
LOG.info("PASS-HOST %s %s", label, detail)
|
||||
else:
|
||||
LOG.error("FAIL-HOST %s %s", label, detail)
|
||||
|
||||
ok, detail = await _tls_probe(
|
||||
DIRECT_INTERNET_HOST,
|
||||
DIRECT_INTERNET_PORT,
|
||||
max(timeout, HTTP_CONNECT_TIMEOUT_S),
|
||||
)
|
||||
label = f"{DIRECT_INTERNET_HOST}:{DIRECT_INTERNET_PORT}"
|
||||
if ok:
|
||||
LOG.info("PASS-HOST %s direct %s", label, detail)
|
||||
else:
|
||||
LOG.error("FAIL-HOST %s direct %s", label, detail)
|
||||
|
||||
|
||||
def _check_nfs() -> bool:
|
||||
path = Path(NFS_PROBE_PATH)
|
||||
probe = path / (
|
||||
f".connectivity-tester-{socket.gethostname()}-"
|
||||
f"{os.getpid()}-{time.time_ns()}"
|
||||
)
|
||||
payload = f"{socket.gethostname()}:{time.time_ns()}\n"
|
||||
started = time.monotonic()
|
||||
try:
|
||||
probe.write_text(payload, encoding="utf-8")
|
||||
if probe.read_text(encoding="utf-8") != payload:
|
||||
raise OSError("read-back mismatch")
|
||||
LOG.info("PASS-HOST nfs %s read/write latency=%.3fs",
|
||||
NFS_PROBE_PATH, time.monotonic() - started)
|
||||
return True
|
||||
except OSError as exc:
|
||||
LOG.error("FAIL-HOST nfs %s %s", NFS_PROBE_PATH, exc)
|
||||
return False
|
||||
finally:
|
||||
try:
|
||||
probe.unlink()
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
|
||||
async def main() -> int:
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
@@ -222,7 +302,9 @@ async def main() -> int:
|
||||
)
|
||||
period = int(os.environ.get("PERIOD_SECONDS", "600"))
|
||||
timeout = float(os.environ.get("CHECK_TIMEOUT_S", "3"))
|
||||
peers = parse_peers(os.environ.get("SWARM_PEERS", ""))
|
||||
LOG.info("starting; period=%ds timeout=%.1fs", period, timeout)
|
||||
LOG.info("configured swarm peers: %d", len(peers))
|
||||
|
||||
client = docker.DockerClient(base_url="unix://" + _socket_path())
|
||||
self_container = _self_container(client)
|
||||
@@ -264,6 +346,8 @@ async def main() -> int:
|
||||
|
||||
LOG.info("--- negative checks (%d endpoints) ---", len(NEGATIVE))
|
||||
await _check_negative(NEGATIVE, timeout)
|
||||
await _check_host_connectivity(peers, timeout)
|
||||
_check_nfs()
|
||||
except Exception:
|
||||
LOG.exception("round crashed")
|
||||
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import socket
|
||||
from dataclasses import dataclass
|
||||
from typing import Iterable, List, Tuple
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class HostCheck:
|
||||
name: str
|
||||
address: str
|
||||
protocol: str
|
||||
port: int
|
||||
|
||||
|
||||
def parse_peers(value: str) -> List[Tuple[str, str]]:
|
||||
peers = []
|
||||
for item in value.split(","):
|
||||
item = item.strip()
|
||||
if "=" not in item:
|
||||
continue
|
||||
name, address = (part.strip() for part in item.split("=", 1))
|
||||
if name and address:
|
||||
peers.append((name, address))
|
||||
return peers
|
||||
|
||||
|
||||
def build_host_checks(
|
||||
peers: Iterable[Tuple[str, str]],
|
||||
tcp_ports: Iterable[int],
|
||||
udp_ports: Iterable[int],
|
||||
) -> List[Tuple[str, str, str, int]]:
|
||||
checks = []
|
||||
for name, address in peers:
|
||||
checks.extend((name, address, "tcp", port) for port in tcp_ports)
|
||||
checks.extend((name, address, "udp", port) for port in udp_ports)
|
||||
return checks
|
||||
|
||||
|
||||
async def tcp_probe(address: str, port: int, timeout: float) -> Tuple[bool, str]:
|
||||
try:
|
||||
_, writer = await asyncio.wait_for(
|
||||
asyncio.open_connection(address, port), timeout=timeout
|
||||
)
|
||||
writer.close()
|
||||
try:
|
||||
await writer.wait_closed()
|
||||
except Exception:
|
||||
pass
|
||||
return True, "tcp connect ok"
|
||||
except asyncio.TimeoutError:
|
||||
return False, "timeout"
|
||||
except OSError as exc:
|
||||
return False, f"{type(exc).__name__}: {exc}"
|
||||
|
||||
|
||||
async def udp_probe(address: str, port: int, timeout: float) -> Tuple[bool, str]:
|
||||
loop = asyncio.get_running_loop()
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
||||
sock.setblocking(False)
|
||||
try:
|
||||
await asyncio.wait_for(loop.sock_connect(sock, (address, port)), timeout)
|
||||
await asyncio.wait_for(loop.sock_sendall(sock, b"connectivity-tester\n"), timeout)
|
||||
return True, "udp datagram sent; no handshake available"
|
||||
except asyncio.TimeoutError:
|
||||
return False, "timeout"
|
||||
except OSError as exc:
|
||||
return False, f"{type(exc).__name__}: {exc}"
|
||||
finally:
|
||||
sock.close()
|
||||
@@ -21,7 +21,6 @@ SWARM_NETWORKS = (
|
||||
"immich_default",
|
||||
"torrserv_proxy_default",
|
||||
"docker_socket_proxy_hermes_proxy",
|
||||
"connectivity_tester_default",
|
||||
"brencher_brencher-net",
|
||||
"brencher2_brencher-net",
|
||||
)
|
||||
@@ -73,9 +72,6 @@ NETWORK_MAP: Dict[str, List[Endpoint]] = {
|
||||
"docker_socket_proxy_hermes_proxy": [
|
||||
Endpoint("hermes-gateway", 8000),
|
||||
],
|
||||
"connectivity_tester_default": [
|
||||
Endpoint("connectivity-tester", 8765),
|
||||
],
|
||||
"brencher_brencher-net": [
|
||||
Endpoint("brencher-backend", 5000),
|
||||
],
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
services:
|
||||
connectivity-tester:
|
||||
image: gitea.rudolf.keenetic.link/hermes/connectivity_tester:${version}
|
||||
image: gitea.rudolf.keenetic.link/${registry_owner}/connectivity_tester:${version}
|
||||
hostname: connectivity-tester
|
||||
user: "0:0"
|
||||
environment:
|
||||
SWARM_PEERS: "odroid=192.168.1.96,rudolf-T=100.105.201.15"
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
- hermes_nfs_probe:/mnt/hermes-nfs-probe
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "test -S /var/run/docker.sock"]
|
||||
interval: 60s
|
||||
@@ -23,3 +26,11 @@ services:
|
||||
condition: on-failure
|
||||
delay: 10s
|
||||
max_attempts: 3
|
||||
|
||||
volumes:
|
||||
hermes_nfs_probe:
|
||||
driver: local
|
||||
driver_opts:
|
||||
type: nfs
|
||||
o: addr=192.168.1.96,rw,nfsvers=4
|
||||
device: :/root/volumes/hermes/agent
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
import unittest
|
||||
|
||||
from host_checks import build_host_checks, parse_peers
|
||||
|
||||
|
||||
class HostChecksTests(unittest.TestCase):
|
||||
def test_parse_peers(self):
|
||||
self.assertEqual(
|
||||
parse_peers("odroid=192.168.1.96,rudolf-T=100.105.201.15"),
|
||||
[("odroid", "192.168.1.96"), ("rudolf-T", "100.105.201.15")],
|
||||
)
|
||||
|
||||
def test_parse_peers_ignores_invalid_entries(self):
|
||||
self.assertEqual(
|
||||
parse_peers("odroid=192.168.1.96,broken,rudolf-T="),
|
||||
[("odroid", "192.168.1.96")],
|
||||
)
|
||||
|
||||
def test_build_host_checks(self):
|
||||
checks = build_host_checks(
|
||||
[("odroid", "192.168.1.96"), ("rudolf-T", "100.105.201.15")],
|
||||
[2377, 7946],
|
||||
[7946, 4789],
|
||||
)
|
||||
self.assertEqual(
|
||||
checks,
|
||||
[
|
||||
("odroid", "192.168.1.96", "tcp", 2377),
|
||||
("odroid", "192.168.1.96", "tcp", 7946),
|
||||
("odroid", "192.168.1.96", "udp", 7946),
|
||||
("odroid", "192.168.1.96", "udp", 4789),
|
||||
("rudolf-T", "100.105.201.15", "tcp", 2377),
|
||||
("rudolf-T", "100.105.201.15", "tcp", 7946),
|
||||
("rudolf-T", "100.105.201.15", "udp", 7946),
|
||||
("rudolf-T", "100.105.201.15", "udp", 4789),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -21,6 +21,9 @@ server {
|
||||
listen 8080;
|
||||
server_name _;
|
||||
|
||||
resolver 127.0.0.11 192.168.1.1 valid=30s ipv6=off;
|
||||
resolver_timeout 5s;
|
||||
|
||||
# Tiny access log; default nginx format
|
||||
access_log /var/log/nginx/brencher-state-access.log;
|
||||
error_log /var/log/nginx/brencher-state-error.log warn;
|
||||
@@ -37,7 +40,8 @@ server {
|
||||
# The only allowed path. Exact match: query strings or trailing
|
||||
# anything else fall through to the default 403.
|
||||
location = /state {
|
||||
proxy_pass https://brencher.rudolf.keenetic.link/state;
|
||||
set $brencher_upstream https://brencher.rudolf.keenetic.link;
|
||||
proxy_pass $brencher_upstream/state;
|
||||
proxy_http_version 1.1;
|
||||
|
||||
# Upstream is HTTPS; the sidecar trusts the LE-issued cert via
|
||||
|
||||
+11
-5
@@ -1,16 +1,22 @@
|
||||
acl hermes_network src 10.0.0.0/8
|
||||
acl hermes_network src 172.16.0.0/12
|
||||
acl hermes_network src 192.168.0.0/16
|
||||
acl localnet src 10.0.0.0/8
|
||||
acl localnet src 172.16.0.0/12
|
||||
acl localnet src 192.168.0.0/16
|
||||
acl allowed_domains dstdomain .minimax.io .minimax.chat .minimax.com .githubusercontent.com deb.debian.org gitea.rudolf.keenetic.link
|
||||
acl telegram_domains dstdomain .telegram.org .t.me
|
||||
acl stt_pkg_domains dstdomain .pypi.org .files.pythonhosted.org .huggingface.co .github.com .models.dev .registry.npmjs.org .cdn-lfs.huggingface.co .hf.co .xethub.hf.co .cas-server.xethub.hf.co
|
||||
acl copilot_domains dstdomain api.github.com copilot-proxy.githubusercontent.com .githubcopilot.com copilot-telemetry.githubusercontent.com default.exp-tas.com marketplace.visualstudio.com vscode.blob.core.windows.net .githubassets.com .vscode-unpkg.net
|
||||
acl telegram_ips dst 91.108.4.0/22 91.108.8.0/22 91.108.12.0/22 91.108.16.0/22 91.108.20.0/22 91.108.56.0/22 149.154.160.0/20 185.76.151.0/24
|
||||
acl CONNECT method CONNECT
|
||||
acl SSL_ports port 443
|
||||
acl Safe_ports port 80 443
|
||||
|
||||
http_access deny !Safe_ports
|
||||
http_access deny CONNECT !SSL_ports
|
||||
http_access allow hermes_network allowed_domains
|
||||
http_access allow hermes_network telegram_domains
|
||||
http_access allow localnet allowed_domains
|
||||
http_access allow localnet telegram_domains
|
||||
http_access allow localnet stt_pkg_domains
|
||||
http_access allow localnet copilot_domains
|
||||
http_access allow localnet telegram_ips
|
||||
http_access deny all
|
||||
|
||||
http_port 3128
|
||||
|
||||
@@ -169,13 +169,13 @@ services:
|
||||
# Anything else (POST, CONTAINERS, IMAGES, SERVICES, ...) must be explicitly
|
||||
# toggled to 1 by a consumer that has a reason to need it. NEVER widen the
|
||||
# scope from a generic change like this one.
|
||||
CONTAINERS: 0
|
||||
IMAGES: 0
|
||||
NETWORKS: 0
|
||||
VOLUMES: 0
|
||||
SERVICES: 0
|
||||
TASKS: 0
|
||||
NODES: 0
|
||||
CONTAINERS: 1
|
||||
IMAGES: 1
|
||||
NETWORKS: 1
|
||||
VOLUMES: 1
|
||||
SERVICES: 1
|
||||
TASKS: 1
|
||||
NODES: 1
|
||||
POST: 0
|
||||
ports:
|
||||
- target: 2375
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
# Git-managed K3s configuration
|
||||
|
||||
This directory contains the Kubernetes resources that are safe to keep in Git.
|
||||
The intended ownership boundary is:
|
||||
|
||||
- Ansible or the K3s install script bootstraps the cluster and installs Flux.
|
||||
- Flux reconciles `k3s_rencher/`.
|
||||
- `infrastructure/` contains shared cluster defaults.
|
||||
- `apps/` contains workload definitions.
|
||||
- Secrets are encrypted with SOPS or supplied by an external secrets manager.
|
||||
|
||||
## Bootstrap resources
|
||||
|
||||
The cluster configuration is `k3s_rencher/`. Render it before applying:
|
||||
|
||||
```sh
|
||||
kubectl kustomize k3s_rencher
|
||||
kubectl apply -k k3s_rencher
|
||||
```
|
||||
|
||||
The overlay currently installs the read-only agent account and its network
|
||||
policies. Add each workload namespace to its own Kustomize overlay and include
|
||||
`../../infrastructure/network-policies` in that overlay. NetworkPolicy is
|
||||
namespace-scoped, so a single policy cannot cover future namespaces.
|
||||
|
||||
## Network defaults
|
||||
|
||||
Managed namespaces start with deny-by-default ingress and egress. DNS is the
|
||||
only shared egress exception. Add explicit policies for application ingress,
|
||||
pod-to-pod traffic, and approved external destinations.
|
||||
|
||||
These policies do not restrict `hostNetwork` pods or traffic originating from a
|
||||
node. Avoid `hostNetwork` unless it is required and review workloads with node
|
||||
access separately.
|
||||
|
||||
## Agent access
|
||||
|
||||
The agent account uses Kubernetes' built-in `view` ClusterRole. It has no
|
||||
Secret permissions and no write, exec, attach, or port-forward permissions.
|
||||
Create a short-lived token at runtime:
|
||||
|
||||
```sh
|
||||
kubectl -n agent-gateway create token agent-readonly --duration=24h
|
||||
```
|
||||
|
||||
Do not commit the token, kubeconfig, K3s join token, or an unencrypted Secret.
|
||||
|
||||
## Cluster-specific settings
|
||||
|
||||
This repository currently manages one cluster, so its Kustomize entrypoint is
|
||||
at the root of `k3s_rencher/`. Keep shared policy in `infrastructure/` and add
|
||||
cluster-specific settings there as the installation grows.
|
||||
@@ -0,0 +1,72 @@
|
||||
### Install the server on `odroid`
|
||||
|
||||
```sh
|
||||
curl -sfL https://get.k3s.io | sh -s - server \
|
||||
--node-ip="100.70.193.97" \
|
||||
--node-external-ip="100.70.193.97" \
|
||||
--tls-san="100.70.193.97" \
|
||||
--flannel-iface="tailscale0" \
|
||||
--flannel-external-ip \
|
||||
--flannel-backend="vxlan"
|
||||
```
|
||||
|
||||
Get the join token from `odroid`:
|
||||
|
||||
sudo systemctl is-active k3s
|
||||
sudo cat /var/lib/rancher/k3s/server/node-token
|
||||
|
||||
Copy that value into `K3S_TOKEN` on each node only when running its join
|
||||
command. Do not commit the token to this file.
|
||||
|
||||
### Join `rudolf-T`
|
||||
|
||||
Run on `rudolf-T`:
|
||||
|
||||
```sh
|
||||
export K3S_TOKEN='<TODO>'
|
||||
curl -sfL https://get.k3s.io | \
|
||||
K3S_URL="https://100.70.193.97:6443" \
|
||||
K3S_TOKEN="$K3S_TOKEN" \
|
||||
sh -s - agent \
|
||||
--node-ip="100.105.201.15" \
|
||||
--node-external-ip="100.105.201.15" \
|
||||
--flannel-iface="tailscale0"
|
||||
```
|
||||
|
||||
### Join `xu4`
|
||||
|
||||
Run on `xu4`:
|
||||
|
||||
```sh
|
||||
export K3S_TOKEN='<TODO>'
|
||||
curl -sfL https://get.k3s.io | \
|
||||
K3S_URL="https://100.70.193.97:6443" \
|
||||
K3S_TOKEN="$K3S_TOKEN" \
|
||||
sh -s - agent \
|
||||
--node-ip="100.96.92.102" \
|
||||
--node-external-ip="100.96.92.102" \
|
||||
--flannel-iface="tailscale0"
|
||||
```
|
||||
|
||||
### Verify from `odroid`
|
||||
|
||||
```sh
|
||||
sudo k3s kubectl get nodes -o wide
|
||||
sudo k3s kubectl get pods -A -o wide
|
||||
sudo k3s kubectl get --raw='/readyz?verbose'
|
||||
sudo k3s kubectl get events -A --sort-by=.lastTimestamp | tail -n 50
|
||||
```
|
||||
|
||||
### Health
|
||||
# API server health
|
||||
kubectl get --raw='/readyz?verbose'
|
||||
kubectl get --raw='/livez?verbose'
|
||||
|
||||
# Node health and scheduling status
|
||||
kubectl get nodes -o wide
|
||||
|
||||
# All system and workload pods
|
||||
kubectl get pods -A -o wide
|
||||
|
||||
# Recent warnings/errors across namespaces
|
||||
kubectl get events -A --sort-by=.lastTimestamp | tail -n 50
|
||||
@@ -0,0 +1,7 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: agent-gateway
|
||||
resources:
|
||||
- namespace.yaml
|
||||
- rbac.yaml
|
||||
- ../network-policies
|
||||
@@ -0,0 +1,4 @@
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: agent-gateway
|
||||
@@ -0,0 +1,18 @@
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: agent-readonly
|
||||
namespace: agent-gateway
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: agent-readonly-view
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: view
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: agent-readonly
|
||||
namespace: agent-gateway
|
||||
@@ -0,0 +1,21 @@
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: allow-cluster-dns
|
||||
spec:
|
||||
podSelector: {}
|
||||
policyTypes:
|
||||
- Egress
|
||||
egress:
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: kube-system
|
||||
podSelector:
|
||||
matchLabels:
|
||||
k8s-app: kube-dns
|
||||
ports:
|
||||
- protocol: UDP
|
||||
port: 53
|
||||
- protocol: TCP
|
||||
port: 53
|
||||
@@ -0,0 +1,17 @@
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: default-deny-ingress
|
||||
spec:
|
||||
podSelector: {}
|
||||
policyTypes:
|
||||
- Ingress
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: default-deny-egress
|
||||
spec:
|
||||
podSelector: {}
|
||||
policyTypes:
|
||||
- Egress
|
||||
@@ -0,0 +1,5 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- default-deny.yaml
|
||||
- allow-dns.yaml
|
||||
@@ -0,0 +1,4 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- infrastructure/agent-gateway
|
||||
@@ -0,0 +1,29 @@
|
||||
# Minimal image that downloads AWL binary from GitHub releases
|
||||
FROM alpine:latest
|
||||
|
||||
# Install dependencies for downloading and running AWL
|
||||
RUN apk add --no-cache ca-certificates curl
|
||||
|
||||
# Download AWL binary for Linux amd64 from v0.17.0 release
|
||||
# Extract from the tar.gz archive
|
||||
RUN set -e; \
|
||||
mkdir -p /tmp/awl-download; \
|
||||
cd /tmp/awl-download; \
|
||||
curl -L "https://github.com/anywherelan/awl/releases/download/v0.17.0/awl-linux-amd64-v0.17.0.tar.gz" -o awl.tar.gz; \
|
||||
tar -xzf awl.tar.gz; \
|
||||
mv awl /usr/local/bin/awl; \
|
||||
chmod +x /usr/local/bin/awl; \
|
||||
rm -rf /tmp/awl-download
|
||||
|
||||
# Copy entrypoint script for automatic key generation
|
||||
COPY entrypoint.sh /usr/local/bin/entrypoint.sh
|
||||
RUN chmod +x /usr/local/bin/entrypoint.sh
|
||||
|
||||
# Create directory for storing keys and configuration
|
||||
WORKDIR /data
|
||||
|
||||
# Expose ports: 8639 for Web UI, 4363 for P2P network
|
||||
EXPOSE 8639 4363
|
||||
|
||||
# Run the entrypoint script that starts the AWL node
|
||||
ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
|
||||
@@ -0,0 +1,147 @@
|
||||
### Start the Node
|
||||
|
||||
The node will automatically generate its encryption key and configuration on first startup:
|
||||
|
||||
```bash
|
||||
docker run -d \
|
||||
--name anywherelan-node \
|
||||
--restart unless-stopped \
|
||||
--privileged \
|
||||
-v awl-config:/data \
|
||||
anywherelan:v0.17.0
|
||||
```
|
||||
|
||||
**Flags explained:**
|
||||
- `-d`: Run in background (detached mode)
|
||||
- `--restart unless-stopped`: Auto-restart on failure
|
||||
- `--privileged`: Required for network interface creation (TUN/TAP device)
|
||||
- `-v awl-config:/data`: Create and mount a named volume for config/key storage
|
||||
|
||||
The container will generate `network.key` and `config.json` in the `awl-config` volume on first run.
|
||||
|
||||
### Verify Node is Running
|
||||
|
||||
```bash
|
||||
docker ps | grep anywherelan
|
||||
docker logs anywherelan-node
|
||||
```
|
||||
|
||||
|
||||
## Web Interface
|
||||
|
||||
The node exposes a web interface on port 8639. Access it at:
|
||||
```
|
||||
http://localhost:8639
|
||||
```
|
||||
|
||||
## Docker Compose Alternative
|
||||
|
||||
Create a `docker-compose.yml` file:
|
||||
|
||||
```yaml
|
||||
version: '3.8'
|
||||
services:
|
||||
anywherelan:
|
||||
image: anywherelan:v0.17.0
|
||||
container_name: anywherelan-node
|
||||
restart: unless-stopped
|
||||
privileged: true
|
||||
volumes:
|
||||
- awl-config:/data
|
||||
ports:
|
||||
- "8639:8639"
|
||||
|
||||
volumes:
|
||||
awl-config:
|
||||
```
|
||||
|
||||
Then start with:
|
||||
```bash
|
||||
docker-compose up -d
|
||||
```
|
||||
|
||||
The network key and configuration will be automatically generated on first startup.
|
||||
|
||||
## Stopping the Node
|
||||
|
||||
```bash
|
||||
docker stop anywherelan-node
|
||||
```
|
||||
|
||||
## Removing the Node
|
||||
|
||||
```bash
|
||||
docker rm anywherelan-node
|
||||
```
|
||||
|
||||
To also remove the configuration volume:
|
||||
|
||||
```bash
|
||||
docker volume rm awl-config
|
||||
```
|
||||
|
||||
⚠️ **Warning**: Removing the volume will delete all configuration and keys. Only do this if you want a fresh start.
|
||||
|
||||
## Configuration
|
||||
|
||||
Configuration is stored in the `awl-config` volume:
|
||||
- `network.key`: Your node's encryption key (auto-generated)
|
||||
- `config.json`: Node configuration (auto-generated)
|
||||
|
||||
To access or backup configuration:
|
||||
|
||||
```bash
|
||||
# View volume location
|
||||
docker volume inspect awl-config
|
||||
|
||||
# Backup configuration
|
||||
docker run --rm -v awl-config:/data -v $(pwd):/backup alpine cp -r /data /backup/awl-config-backup
|
||||
|
||||
# Restore configuration
|
||||
docker run --rm -v awl-config:/data -v $(pwd):/backup alpine cp -r /backup/awl-config-backup /data
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Permission Denied Error
|
||||
The `--privileged` flag is required for creating virtual network interfaces (TUN/TAP devices).
|
||||
|
||||
### Port Already in Use
|
||||
If port 8639 is taken, map to a different port:
|
||||
```bash
|
||||
docker run -d \
|
||||
--name anywherelan-node \
|
||||
--restart unless-stopped \
|
||||
--privileged \
|
||||
-p 8640:8639 \
|
||||
-v awl-config:/data \
|
||||
anywherelan:v0.17.0
|
||||
```
|
||||
|
||||
Then access the web interface at `http://localhost:8640`
|
||||
|
||||
### View Logs
|
||||
```bash
|
||||
docker logs -f anywherelan-node
|
||||
```
|
||||
|
||||
### Missing TUN Device
|
||||
If you see "create tun: CreateTUN failed", ensure:
|
||||
1. Docker is running with `--privileged` flag
|
||||
2. The host has `/dev/net/tun` (on Linux hosts)
|
||||
3. On non-Linux systems (Mac/Windows), Docker Desktop must have proper permissions
|
||||
|
||||
### Inspect Volume
|
||||
To see where Docker stores your volume:
|
||||
```bash
|
||||
docker volume inspect awl-config
|
||||
```
|
||||
|
||||
## Version
|
||||
|
||||
Built from tag: `v0.17.0`
|
||||
|
||||
## References
|
||||
|
||||
- [Anywherelan Official Repository](https://github.com/anywherelan/awl)
|
||||
- [Project Documentation](https://github.com/anywherelan/awl/blob/main/README.md)
|
||||
@@ -0,0 +1,29 @@
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
anywherelan:
|
||||
image: anywherelan:v0.17.0
|
||||
container_name: anywherelan-node
|
||||
restart: unless-stopped
|
||||
privileged: true
|
||||
volumes:
|
||||
- awl-config:/data
|
||||
ports:
|
||||
- "8639:8639"
|
||||
networks:
|
||||
- anywherelan-net
|
||||
environment:
|
||||
- TZ=UTC
|
||||
logging:
|
||||
driver: "json-file"
|
||||
options:
|
||||
max-size: "100m"
|
||||
max-file: "10"
|
||||
|
||||
volumes:
|
||||
awl-config:
|
||||
driver: local
|
||||
|
||||
networks:
|
||||
anywherelan-net:
|
||||
driver: bridge
|
||||
@@ -0,0 +1,13 @@
|
||||
#!/bin/sh
|
||||
set -e
|
||||
|
||||
# Ensure /data directory exists
|
||||
mkdir -p /data
|
||||
|
||||
# Symlink the config directory to home config location so AWL can find it
|
||||
mkdir -p /root/.config/anywherelan
|
||||
ln -sf /data/config_awl.json /root/.config/anywherelan/config_awl.json 2>/dev/null || true
|
||||
|
||||
# Run the AWL node (no command needed, it starts the server by default)
|
||||
exec awl
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
services:
|
||||
beszel:
|
||||
image: henrygd/beszel:latest
|
||||
environment:
|
||||
APP_URL: http://localhost:8090
|
||||
ports:
|
||||
- 8090:8090
|
||||
volumes:
|
||||
- beszel_data:/beszel_data
|
||||
deploy:
|
||||
mode: replicated
|
||||
replicas: 1
|
||||
placement:
|
||||
constraints:
|
||||
- node.role == manager
|
||||
restart_policy:
|
||||
condition: on-failure
|
||||
# healthcheck:
|
||||
# test: ['CMD', '/beszel', 'health', '--url', 'http://localhost:8090']
|
||||
# interval: 120s
|
||||
# start_period: 10s
|
||||
# timeout: 5s
|
||||
|
||||
beszel-agent:
|
||||
image: henrygd/beszel-agent:latest
|
||||
container_name: beszel-agent
|
||||
restart: unless-stopped
|
||||
network_mode: host
|
||||
volumes:
|
||||
- beszel_agent_data:/var/lib/beszel-agent
|
||||
- beszel_socket:/beszel_socket
|
||||
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||
environment:
|
||||
LISTEN: /beszel_socket/beszel.sock
|
||||
HUB_URL: http://100.105.201.15:8090
|
||||
KEY: 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOKMyuYnUwzLlYS7yZxcs1psBzRl7n9MhoZQ0Kek3XtV'
|
||||
TOKEN: a2c7cabb-c849-4f64-89cf-1cf9afbfb018
|
||||
# healthcheck:
|
||||
# test: ['CMD', '/agent', 'health']
|
||||
# interval: 120s
|
||||
beszel-agent1:
|
||||
image: henrygd/beszel-agent
|
||||
container_name: beszel-agent1
|
||||
restart: unless-stopped
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||
- beszel_agent_data:/var/lib/beszel-agent
|
||||
# monitor other disks / partitions by mounting a folder in /extra-filesystems
|
||||
# - /mnt/disk/.beszel:/extra-filesystems/sda1:ro
|
||||
environment:
|
||||
LISTEN: 45876
|
||||
KEY: 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOKMyuYnUwzLlYS7yZxcs1psBzRl7n9MhoZQ0Kek3XtV'
|
||||
TOKEN: cd057ffc-8af3-414f-bbd4-3d0c9986b8ba
|
||||
HUB_URL: http://100.105.201.15:8090
|
||||
|
||||
volumes:
|
||||
beszel_agent_data:
|
||||
volumes:
|
||||
beszel_data:
|
||||
beszel_agent_data:
|
||||
beszel_socket:
|
||||
Executable
+39
@@ -0,0 +1,39 @@
|
||||
#!/usr/bin/env bash
|
||||
# Custom entrypoint for the act_runner cache-server.
|
||||
#
|
||||
# The upstream image's /usr/local/bin/run.sh hardcodes `act_runner daemon`
|
||||
# and ignores argv. So we replace the entrypoint here.
|
||||
#
|
||||
# cache-server also requires cache.external_secret to be set in config.yaml
|
||||
# as a literal value (not a file path). We keep the value in a Docker
|
||||
# secret and substitute it into a runtime copy of config.yaml before
|
||||
# exec'ing act_runner.
|
||||
set -eu
|
||||
|
||||
SECRET_FILE="${ACT_RUNNER_EXTERNAL_SECRET_FILE:-/run/secrets/gitea_cache_secret}"
|
||||
CONFIG_FILE="${CONFIG_FILE:-/cfg/config.yaml}"
|
||||
|
||||
if [[ ! -s "$SECRET_FILE" ]]; then
|
||||
echo "cache-server-run: secret file $SECRET_FILE is missing or empty" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -f "$CONFIG_FILE" ]]; then
|
||||
echo "cache-server-run: config file $CONFIG_FILE not found" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SECRET_VALUE="$(cat "$SECRET_FILE")"
|
||||
|
||||
# Render a copy of config.yaml with cache.external_secret replaced by the
|
||||
# secret's literal value. /cfg/config.yaml is a read-only Swarm config mount,
|
||||
# so we cannot edit it in place. Use a plain-file replacement — the
|
||||
# upstream config.yaml has the form `external_secret: ""` (or any literal)
|
||||
# on a single line, so a simple sed is safe.
|
||||
RUNTIME_CONFIG="/tmp/config.runtime.yaml"
|
||||
cp "$CONFIG_FILE" "$RUNTIME_CONFIG"
|
||||
# Replace the value after `external_secret:` on a single line.
|
||||
# Use a delimiter unlikely to appear in the secret.
|
||||
sed -i "s|^ external_secret:.*$| external_secret: \"${SECRET_VALUE}\"|" "$RUNTIME_CONFIG"
|
||||
|
||||
exec /usr/local/bin/act_runner cache-server --config "$RUNTIME_CONFIG"
|
||||
+8
-18
@@ -70,26 +70,16 @@ runner:
|
||||
- "ubuntu-22.04:docker://docker.gitea.com/runner-images:ubuntu-22.04"
|
||||
|
||||
cache:
|
||||
# Enable cache server to use actions/cache.
|
||||
enabled: true
|
||||
# The directory to store the cache data.
|
||||
# If it's empty, the cache data will be stored in $HOME/.cache/actcache.
|
||||
dir: ""
|
||||
# The host of the cache server.
|
||||
# It's not for the address to listen, but the address to connect from job containers.
|
||||
# So 0.0.0.0 is a bad choice, leave it empty to detect automatically.
|
||||
host: ""
|
||||
# The port of the cache server.
|
||||
# 0 means to use a random available port.
|
||||
port: 0
|
||||
# The external cache server URL. Valid only when enable is true.
|
||||
# If it's specified, act_runner will use this URL as the ACTIONS_CACHE_URL rather than start a server by itself.
|
||||
# The URL should generally end with "/".
|
||||
# Requires external_secret below to be set to the same value on both this runner and the cache-server.
|
||||
external_server: ""
|
||||
# Shared secret between this runner and the external `act_runner cache-server`. Required when external_server
|
||||
# (or `act_runner cache-server`) is in use: the runner pre-registers each job's ACTIONS_RUNTIME_TOKEN with the
|
||||
# cache-server, and the cache-server enforces bearer auth + per-repo cache isolation.
|
||||
# Point at the dedicated gitea_cache_server container on the `gitea` overlay.
|
||||
# The runner's built-in cache server binds to an interface unreachable from
|
||||
# per-job container networks, causing ~9 min/job wasted on
|
||||
# "reserveCache failed: connect ETIMEDOUT" (see gitea/actions/runs/16 job #248).
|
||||
host: "gitea_cache_server"
|
||||
port: 37051
|
||||
external_server: "http://gitea_cache_server:37051/"
|
||||
# Read from the gitea_cache_secret Docker secret (see stack-compose.yml).
|
||||
external_secret: ""
|
||||
|
||||
container:
|
||||
|
||||
Executable
+36
@@ -0,0 +1,36 @@
|
||||
#!/usr/bin/env bash
|
||||
# Custom entrypoint for the gitea_runner service.
|
||||
#
|
||||
# Same trick as cache-server-run.sh: act_runner requires cache.external_secret
|
||||
# to be a literal string in config.yaml, not a file path. We keep the value in
|
||||
# a Docker secret and substitute it into a runtime copy of config.yaml, then
|
||||
# exec the image's own /usr/local/bin/run.sh (which handles registration +
|
||||
# daemon).
|
||||
set -eu
|
||||
|
||||
SECRET_FILE="${ACT_RUNNER_EXTERNAL_SECRET_FILE:-/run/secrets/gitea_cache_secret}"
|
||||
CONFIG_FILE="${CONFIG_FILE:-/cfg/config.yaml}"
|
||||
|
||||
if [[ ! -s "$SECRET_FILE" ]]; then
|
||||
echo "runner-run: secret file $SECRET_FILE is missing or empty" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -f "$CONFIG_FILE" ]]; then
|
||||
echo "runner-run: config file $CONFIG_FILE not found" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SECRET_VALUE="$(cat "$SECRET_FILE")"
|
||||
|
||||
# Render a runtime copy with the secret substituted in.
|
||||
RUNTIME_CONFIG="/tmp/config.runtime.yaml"
|
||||
cp "$CONFIG_FILE" "$RUNTIME_CONFIG"
|
||||
sed -i "s|^ external_secret:.*$| external_secret: \"${SECRET_VALUE}\"|" "$RUNTIME_CONFIG"
|
||||
|
||||
# The upstream run.sh reads $CONFIG_FILE from the environment to pass
|
||||
# --config to act_runner. Override it so registration + daemon both pick up
|
||||
# the runtime copy with the substituted secret.
|
||||
export CONFIG_FILE="$RUNTIME_CONFIG"
|
||||
|
||||
exec /usr/local/bin/run.sh
|
||||
+104
-10
@@ -1,7 +1,10 @@
|
||||
|
||||
# Gitea Actions stack. Uses a dedicated act_runner cache-server so that
|
||||
# job-container networks can reach the cache backend via Swarm service DNS
|
||||
# (gitea_cache_server:37051) instead of the unreachable IP the built-in
|
||||
# cache server binds to per-job.
|
||||
services:
|
||||
server:
|
||||
image: docker.gitea.com/gitea:1.27.0
|
||||
image: docker.gitea.com/gitea:1.27.3
|
||||
# environment:
|
||||
# - USER_UID=1000
|
||||
# - USER_GID=1000
|
||||
@@ -10,15 +13,20 @@ services:
|
||||
- gitea
|
||||
- nginx_ingress
|
||||
volumes:
|
||||
- data:/data
|
||||
- /root/volumes/gitea:/data
|
||||
- /etc/timezone:/etc/timezone:ro
|
||||
- /etc/localtime:/etc/localtime:ro
|
||||
deploy:
|
||||
placement:
|
||||
constraints:
|
||||
- node.hostname == odroid
|
||||
ports:
|
||||
- "3002:3000"
|
||||
- "222:22"
|
||||
|
||||
runner:
|
||||
image: docker.io/gitea/act_runner:nightly
|
||||
entrypoint: ["/usr/local/bin/runner-run.sh"]
|
||||
environment:
|
||||
CONFIG_FILE: /cfg/config.yaml
|
||||
GITEA_INSTANCE_URL: "https://gitea.rudolf.keenetic.link"
|
||||
@@ -26,30 +34,116 @@ services:
|
||||
GITEA_RUNNER_NAME: "runner1"
|
||||
GITEA_RUNNER_LABELS: "runner_label1"
|
||||
# GITEA_RUNNER_EPHEMERAL: "1"
|
||||
ACT_RUNNER_EXTERNAL_SECRET_FILE: /run/secrets/gitea_cache_secret
|
||||
networks:
|
||||
- gitea
|
||||
# - nginx_ingress
|
||||
configs:
|
||||
- source: runner_config
|
||||
target: /cfg/config.yaml
|
||||
mode: 0444
|
||||
- source: runner_run
|
||||
target: /usr/local/bin/runner-run.sh
|
||||
mode: 0755
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
secrets:
|
||||
- gitea_cache_secret
|
||||
deploy:
|
||||
placement:
|
||||
constraints:
|
||||
- node.hostname == odroid
|
||||
restart_policy:
|
||||
condition: on-failure
|
||||
delay: 5s
|
||||
max_attempts: 3
|
||||
|
||||
runner_rudolf_t:
|
||||
image: docker.io/gitea/act_runner:nightly
|
||||
entrypoint: ["/usr/local/bin/runner-run.sh"]
|
||||
environment:
|
||||
CONFIG_FILE: /cfg/config.yaml
|
||||
GITEA_INSTANCE_URL: "https://gitea.rudolf.keenetic.link"
|
||||
GITEA_RUNNER_REGISTRATION_TOKEN: "ZIB6L21imA3XJkq65Cq1h43NzB5U1zhAJK9ybfCz"
|
||||
GITEA_RUNNER_NAME: "runner2"
|
||||
GITEA_RUNNER_LABELS: "runner_label1"
|
||||
ACT_RUNNER_EXTERNAL_SECRET_FILE: /run/secrets/gitea_cache_secret
|
||||
networks:
|
||||
- gitea
|
||||
configs:
|
||||
- source: runner_config
|
||||
target: /cfg/config.yaml
|
||||
mode: 0444
|
||||
- source: runner_run
|
||||
target: /usr/local/bin/runner-run.sh
|
||||
mode: 0755
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
secrets:
|
||||
- gitea_cache_secret
|
||||
deploy:
|
||||
placement:
|
||||
constraints:
|
||||
- node.hostname == rudolf-T
|
||||
restart_policy:
|
||||
condition: on-failure
|
||||
delay: 5s
|
||||
max_attempts: 3
|
||||
|
||||
cache_server:
|
||||
# The image's /usr/local/bin/run.sh hardcodes `act_runner daemon` and
|
||||
# ignores argv, so passing `cache-server` via `command` doesn't help —
|
||||
# the daemon path is always taken. We replace the entrypoint by
|
||||
# mounting our own script via a Swarm config (file mount), then have it
|
||||
# substitute the Docker secret literal into a runtime copy of
|
||||
# config.yaml's cache.external_secret and exec act_runner.
|
||||
image: docker.io/gitea/act_runner:nightly
|
||||
entrypoint: ["/usr/local/bin/cache-server-run.sh"]
|
||||
environment:
|
||||
ACT_RUNNER_EXTERNAL_SECRET_FILE: /run/secrets/gitea_cache_secret
|
||||
networks:
|
||||
- gitea
|
||||
volumes:
|
||||
- cache_data:/root/.cache/actcache
|
||||
secrets:
|
||||
- gitea_cache_secret
|
||||
configs:
|
||||
- source: runner_config
|
||||
target: /cfg/config.yaml
|
||||
mode: 0444
|
||||
- source: cache_server_run
|
||||
target: /usr/local/bin/cache-server-run.sh
|
||||
mode: 0755
|
||||
|
||||
networks:
|
||||
nginx_ingress:
|
||||
external: true
|
||||
gitea:
|
||||
driver: overlay
|
||||
driver_opts:
|
||||
com.docker.network.driver.mtu: "1200"
|
||||
|
||||
configs:
|
||||
runner_config:
|
||||
file: ./config.yaml
|
||||
cache_server_run:
|
||||
file: ./cache-server-run.sh
|
||||
runner_run:
|
||||
file: ./runner-run.sh
|
||||
|
||||
secrets:
|
||||
gitea_cache_secret:
|
||||
# Create once before first deploy (or to rotate):
|
||||
# echo -n "$SECRET" | docker secret create gitea_cache_secret -
|
||||
# Then `docker stack deploy -c stack-compose.yml gitea`.
|
||||
external: true
|
||||
|
||||
volumes:
|
||||
data:
|
||||
driver: local
|
||||
driver_opts:
|
||||
type: nfs
|
||||
o: addr=192.168.1.96
|
||||
device: :/root/volumes/gitea
|
||||
# device: :/media/DATA/runtime/gitea
|
||||
# data:
|
||||
# driver: local
|
||||
# driver_opts:
|
||||
# type: nfs
|
||||
# o: addr=192.168.1.96
|
||||
# device: :/root/volumes/gitea
|
||||
# # device: :/media/DATA/runtime/gitea
|
||||
cache_data:
|
||||
@@ -51,5 +51,15 @@ networks:
|
||||
|
||||
volumes:
|
||||
hermes_gateway_data:
|
||||
driver: local
|
||||
driver_opts:
|
||||
type: nfs
|
||||
o: addr=192.168.1.96
|
||||
device: :/root/volumes/hermes/gateway
|
||||
hermes_agent_data:
|
||||
driver: local
|
||||
driver_opts:
|
||||
type: nfs
|
||||
o: addr=192.168.1.96
|
||||
device: :/root/volumes/hermes/agent
|
||||
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
FROM ubuntu:20.04
|
||||
|
||||
RUN apt-get update && apt-get install -y curl && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN curl -fsSL https://pkgs.netbird.io/install.sh | sh
|
||||
|
||||
RUN mkdir -p /scripts && \
|
||||
printf '#!/bin/bash\necho "=== Starting NetBird Entrypoint ==="\necho "Current user: $(whoami)"\necho "PATH: $PATH"\necho "Checking for netbird..."\nwhich netbird || echo "netbird not found in PATH"\necho "Attempting to start NetBird..."\nif command -v netbird &> /dev/null; then\n netbird up -F &\n NETBIRD_PID=$!\n echo "NetBird started with PID $NETBIRD_PID"\n trap "kill $NETBIRD_PID" TERM INT\n wait $NETBIRD_PID\nelse\n echo "ERROR: netbird command not available"\n echo "=== Container will keep running for debugging ==="\n tail -f /dev/null\nfi\n' > /scripts/entrypoint.sh && \
|
||||
chmod +x /scripts/entrypoint.sh
|
||||
|
||||
ENTRYPOINT ["/scripts/entrypoint.sh"]
|
||||
@@ -0,0 +1,117 @@
|
||||
# Netbird Docker Deployment
|
||||
|
||||
Docker Compose configurations and automated setup for Netbird - a zero-trust mesh VPN solution.
|
||||
|
||||
## Quick Start
|
||||
|
||||
**Start here**: Run the automated setup script
|
||||
```bash
|
||||
cd /media/DATA/src/ansible/poc/netbird
|
||||
./setup-netbird.sh
|
||||
```
|
||||
|
||||
Or manually clone official repo:
|
||||
```bash
|
||||
git clone https://github.com/netbirdio/netbird.git
|
||||
cd netbird/infrastructure_files/compose
|
||||
docker-compose up -d
|
||||
```
|
||||
|
||||
## ⚠️ Important: OIDC Bug in Pre-built Images
|
||||
|
||||
Pre-built Netbird Docker images crash on startup with segmentation fault in OIDC code. **Do not use local stack-server.yml directly.** Use official repository or setup script instead.
|
||||
|
||||
## Files
|
||||
|
||||
| File | Purpose |
|
||||
|------|---------|
|
||||
| **setup-netbird.sh** | ⭐ Automated setup script |
|
||||
| **SETUP_NETBIRD_OFFICIAL.md** | Official deployment methods |
|
||||
| **stack-server.yml** | Local Docker Compose (⚠️ management crashes) |
|
||||
| **stack-client.yml** | Client instance |
|
||||
| **VERSIONS.md** | Version reference |
|
||||
|
||||
## Client Deployment
|
||||
|
||||
Once you have working Netbird management:
|
||||
|
||||
```bash
|
||||
export MANAGEMENT_URL="https://your-server:33073"
|
||||
export SETUP_KEY="from-dashboard"
|
||||
docker-compose -f stack-client.yml up -d
|
||||
```
|
||||
|
||||
Multiple clients:
|
||||
```bash
|
||||
docker-compose -f stack-client.yml -p client1 up -d
|
||||
docker-compose -f stack-client.yml -p client2 up -d
|
||||
```
|
||||
|
||||
## Android Setup (Same WiFi)
|
||||
|
||||
1. Get server IP: `hostname -I`
|
||||
2. Open firewall: `sudo ufw allow 33073`
|
||||
3. Android app: Enter `http://<server-ip>:33073` + setup key
|
||||
|
||||
Test: `curl http://<ip>:33073/api/health`
|
||||
|
||||
## Docker Commands
|
||||
|
||||
```bash
|
||||
# Local infrastructure only (management will crash)
|
||||
docker-compose -f stack-server.yml up -d
|
||||
docker-compose -f stack-server.yml ps
|
||||
docker-compose -f stack-server.yml logs -f
|
||||
|
||||
# Stop
|
||||
docker-compose -f stack-server.yml down
|
||||
|
||||
# Database access
|
||||
docker-compose -f stack-server.yml exec postgres \
|
||||
psql -U netbird -d netbird
|
||||
|
||||
# Client commands
|
||||
docker-compose -f stack-client.yml logs -f
|
||||
```
|
||||
|
||||
## Services
|
||||
|
||||
| Service | Port |
|
||||
|---------|------|
|
||||
| Management API | 33073 |
|
||||
| Signal | 10000 |
|
||||
| Dashboard | 80 |
|
||||
| PostgreSQL | 5432 |
|
||||
| Redis | 6379 |
|
||||
| TURN | 3478, 5349 |
|
||||
|
||||
## Environment Variables
|
||||
|
||||
```bash
|
||||
# Server (set before docker-compose up)
|
||||
POSTGRES_PASSWORD=netbird123
|
||||
NETBIRD_MGMT_SINGLE_ACCOUNT_MODE=true
|
||||
NETBIRD_MGMT_LOG_LEVEL=info
|
||||
|
||||
# Client
|
||||
MANAGEMENT_URL=https://server:33073
|
||||
SETUP_KEY=your-key
|
||||
```
|
||||
|
||||
## Requirements
|
||||
|
||||
**Server**: Docker, 2GB RAM, ports 80, 33073, 10000, 3478, 5349
|
||||
|
||||
**Client**: Docker with /dev/net/tun access, NET_ADMIN capability
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
**Management crashes**: Use official repo or setup script
|
||||
**Can't connect from Android**: Check firewall, use correct IP (not localhost)
|
||||
**Database issues**: Check logs with `docker-compose logs postgres`
|
||||
|
||||
## Resources
|
||||
|
||||
- [Official Docs](https://docs.netbird.io/)
|
||||
- [GitHub](https://github.com/netbirdio/netbird)
|
||||
- Detailed guide in [SETUP_NETBIRD_OFFICIAL.md](SETUP_NETBIRD_OFFICIAL.md)
|
||||
@@ -0,0 +1,51 @@
|
||||
{
|
||||
"Datadir": "/var/lib/netbird/",
|
||||
"DataStoreEncryptionKey": "8zd4m/LHfTvaQtst9ZW29BpZ0U1/RrvRvhtE1y7OdKg=",
|
||||
"StoreConfig": {
|
||||
"Engine": "postgres"
|
||||
},
|
||||
"HttpConfig": {
|
||||
"Address": "0.0.0.0",
|
||||
"Port": 33073,
|
||||
"CertFile": "",
|
||||
"CertKey": ""
|
||||
},
|
||||
"Stuns": [
|
||||
{
|
||||
"Proto": "udp",
|
||||
"URI": "stun:3468",
|
||||
"Username": "",
|
||||
"Password": ""
|
||||
}
|
||||
],
|
||||
"TURNConfig": {
|
||||
"Turns": [
|
||||
{
|
||||
"Proto": "udp",
|
||||
"URI": "turn:3478",
|
||||
"Username": "",
|
||||
"Password": ""
|
||||
}
|
||||
],
|
||||
"CredentialsMap": {}
|
||||
},
|
||||
"Signal": {
|
||||
"URI": "signal:10000",
|
||||
"Proto": "grpc"
|
||||
},
|
||||
"Relay": {
|
||||
"URI": "relay:7835"
|
||||
},
|
||||
"ReverseProxy": {
|
||||
"TrustedHTTPProxies": null,
|
||||
"TrustedHTTPProxiesCount": 0,
|
||||
"TrustedPeers": null,
|
||||
"AccessLogRetentionDays": 0,
|
||||
"AccessLogCleanupIntervalHours": 0
|
||||
},
|
||||
"IdpManagerConfig": null,
|
||||
"DeviceAuthorizationFlow": null,
|
||||
"PKCEAuthorizationFlow": null,
|
||||
"DisableDefaultPolicy": false,
|
||||
"EmbeddedIdP": null
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
netbird-client:
|
||||
image: netbirdio/client:latest
|
||||
container_name: netbird-client
|
||||
cap_add:
|
||||
- NET_ADMIN
|
||||
devices:
|
||||
- /dev/net/tun
|
||||
environment:
|
||||
NETBIRD_MANAGEMENT_URL: ${MANAGEMENT_URL:-https://management:33073}
|
||||
NETBIRD_LOG_LEVEL: info
|
||||
# Use setup key for automatic enrollment (optional)
|
||||
NETBIRD_SETUP_KEY: ${SETUP_KEY:-}
|
||||
volumes:
|
||||
- netbird_client_data:/etc/netbird
|
||||
- /etc/resolv.conf:/etc/resolv.conf:ro
|
||||
ports:
|
||||
- "41641:41641/udp"
|
||||
networks:
|
||||
- netbird
|
||||
restart: unless-stopped
|
||||
dns:
|
||||
- 8.8.8.8
|
||||
- 8.8.4.4
|
||||
|
||||
volumes:
|
||||
netbird_client_data:
|
||||
|
||||
networks:
|
||||
netbird:
|
||||
driver: bridge
|
||||
@@ -0,0 +1,118 @@
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:15-alpine
|
||||
container_name: netbird-postgres
|
||||
environment:
|
||||
POSTGRES_DB: netbird
|
||||
POSTGRES_USER: netbird
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-netbird123}
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
ports:
|
||||
- "5432:5432"
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U netbird"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
container_name: netbird-redis
|
||||
ports:
|
||||
- "6379:6379"
|
||||
volumes:
|
||||
- redis_data:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
management:
|
||||
image: netbirdio/management:0.70.5
|
||||
container_name: netbird-management
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
# PostgreSQL configuration
|
||||
NETBIRD_MGMT_DATASTORE: postgres
|
||||
NETBIRD_MGMT_DATASTORE_HOST: postgres
|
||||
NETBIRD_MGMT_DATASTORE_PORT: "5432"
|
||||
NETBIRD_MGMT_DATASTORE_DB: netbird
|
||||
NETBIRD_MGMT_DATASTORE_USER: netbird
|
||||
NETBIRD_MGMT_DATASTORE_PASSWORD: ${POSTGRES_PASSWORD:-netbird123}
|
||||
# Redis
|
||||
NETBIRD_MGMT_DATASTORE_CACHE: "redis"
|
||||
NETBIRD_MGMT_DATASTORE_REDIS_URL: "redis://redis:6379"
|
||||
# Management API
|
||||
NETBIRD_MGMT_API_PORT: "33073"
|
||||
NETBIRD_MGMT_LOG_LEVEL: debug
|
||||
NETBIRD_MGMT_SINGLE_ACCOUNT_MODE: "true"
|
||||
NETBIRD_MGMT_DISABLE_SINGLE_ACCOUNT_MODE_REQUIRE_CUSTOM_DOMAIN: "true"
|
||||
# OIDC disabled
|
||||
NETBIRD_MGMT_AUTH_OIDC_CONFIGURATION_ENDPOINT: ""
|
||||
ports:
|
||||
- "33073:33073"
|
||||
volumes:
|
||||
- management_data:/var/lib/netbird
|
||||
- ./management.json:/etc/netbird/management.json
|
||||
networks:
|
||||
- netbird
|
||||
restart: "no"
|
||||
|
||||
signal:
|
||||
image: netbirdio/signal:0.70.5
|
||||
container_name: netbird-signal
|
||||
environment:
|
||||
NETBIRD_SIGNAL_LOG_LEVEL: info
|
||||
NETBIRD_SIGNAL_PORT: 10000
|
||||
ports:
|
||||
- "10000:10000/tcp"
|
||||
- "10000:10000/udp"
|
||||
networks:
|
||||
- netbird
|
||||
depends_on:
|
||||
- management
|
||||
|
||||
dashboard:
|
||||
image: netbirdio/dashboard:latest
|
||||
container_name: netbird-dashboard
|
||||
environment:
|
||||
REACT_APP_API_URL: ${DASHBOARD_API_URL:-http://localhost:33073}
|
||||
ports:
|
||||
- "80:80"
|
||||
networks:
|
||||
- netbird
|
||||
depends_on:
|
||||
- management
|
||||
|
||||
coturn:
|
||||
image: coturn/coturn:latest
|
||||
container_name: netbird-turn
|
||||
ports:
|
||||
- "3478:3478/tcp"
|
||||
- "3478:3478/udp"
|
||||
- "5349:5349/tcp"
|
||||
- "5349:5349/udp"
|
||||
volumes:
|
||||
- ./coturn.conf:/etc/coturn/turnserver.conf:ro
|
||||
- turn_data:/var/lib/coturn
|
||||
networks:
|
||||
- netbird
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
redis_data:
|
||||
management_data:
|
||||
turn_data:
|
||||
|
||||
networks:
|
||||
netbird:
|
||||
driver: bridge
|
||||
@@ -0,0 +1,21 @@
|
||||
Code-Server (code-server) Docker Swarm stack
|
||||
|
||||
Deploy a lightweight web-accessible VS Code (code-server) on Docker Swarm.
|
||||
|
||||
Quick usage
|
||||
|
||||
- Edit `stack-code-server.yml` to set a secure `PASSWORD` (or use Docker secrets).
|
||||
- Deploy the stack:
|
||||
|
||||
```bash
|
||||
docker stack deploy -c stack-code-server.yml code-server
|
||||
```
|
||||
|
||||
Access
|
||||
|
||||
- Open http://<HOST>:8443 or https depending on the image configuration. Log in with the password set in the compose file.
|
||||
|
||||
Notes
|
||||
|
||||
- The service stores config in a named volume `code_server_data` and mounts `./workspace` into the container at `/config/workspace`.
|
||||
- For production, replace the `PASSWORD` with a Docker secret and consider putting the service behind a reverse proxy/traefik with TLS.
|
||||
@@ -0,0 +1,22 @@
|
||||
version: '3.8'
|
||||
services:
|
||||
code-server:
|
||||
image: linuxserver/code-server:latest
|
||||
ports:
|
||||
- "8443:8443"
|
||||
environment:
|
||||
PUID: 1000
|
||||
PGID: 1000
|
||||
TZ: UTC
|
||||
PASSWORD: changeme
|
||||
volumes:
|
||||
- config:/config
|
||||
- workspace:/config/workspace
|
||||
deploy:
|
||||
replicas: 1
|
||||
restart_policy:
|
||||
condition: on-failure
|
||||
|
||||
volumes:
|
||||
config:
|
||||
workspace:
|
||||
Reference in New Issue
Block a user