Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8b4695733b | ||
|
|
c8e33e2600 | ||
|
|
0a9e795b8f | ||
|
|
db6a79e1d0 | ||
|
|
7e69577609 |
@@ -3,6 +3,8 @@ services:
|
||||
image: gitea.rudolf.keenetic.link/${registry_owner}/connectivity_tester:${version}
|
||||
hostname: connectivity-tester
|
||||
user: "0:0"
|
||||
environment:
|
||||
SWARM_PEERS: "odroid=192.168.1.96,rudolf-T=100.105.201.15"
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
- hermes_nfs_probe:/mnt/hermes-nfs-probe
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
# Git-managed K3s configuration
|
||||
|
||||
This directory contains the Kubernetes resources that are safe to keep in Git.
|
||||
The intended ownership boundary is:
|
||||
|
||||
- Ansible or the K3s install script bootstraps the cluster and installs Flux.
|
||||
- Flux reconciles `k3s_rencher/`.
|
||||
- `infrastructure/` contains shared cluster defaults.
|
||||
- `apps/` contains workload definitions.
|
||||
- Secrets are encrypted with SOPS or supplied by an external secrets manager.
|
||||
|
||||
## Bootstrap resources
|
||||
|
||||
The cluster configuration is `k3s_rencher/`. Render it before applying:
|
||||
|
||||
```sh
|
||||
kubectl kustomize k3s_rencher
|
||||
kubectl apply -k k3s_rencher
|
||||
```
|
||||
|
||||
The overlay currently installs the read-only agent account and its network
|
||||
policies. Add each workload namespace to its own Kustomize overlay and include
|
||||
`../../infrastructure/network-policies` in that overlay. NetworkPolicy is
|
||||
namespace-scoped, so a single policy cannot cover future namespaces.
|
||||
|
||||
## Network defaults
|
||||
|
||||
Managed namespaces start with deny-by-default ingress and egress. DNS is the
|
||||
only shared egress exception. Add explicit policies for application ingress,
|
||||
pod-to-pod traffic, and approved external destinations.
|
||||
|
||||
These policies do not restrict `hostNetwork` pods or traffic originating from a
|
||||
node. Avoid `hostNetwork` unless it is required and review workloads with node
|
||||
access separately.
|
||||
|
||||
## Agent access
|
||||
|
||||
The agent account uses Kubernetes' built-in `view` ClusterRole. It has no
|
||||
Secret permissions and no write, exec, attach, or port-forward permissions.
|
||||
Create a short-lived token at runtime:
|
||||
|
||||
```sh
|
||||
kubectl -n agent-gateway create token agent-readonly --duration=24h
|
||||
```
|
||||
|
||||
Do not commit the token, kubeconfig, K3s join token, or an unencrypted Secret.
|
||||
|
||||
## Cluster-specific settings
|
||||
|
||||
This repository currently manages one cluster, so its Kustomize entrypoint is
|
||||
at the root of `k3s_rencher/`. Keep shared policy in `infrastructure/` and add
|
||||
cluster-specific settings there as the installation grows.
|
||||
@@ -0,0 +1,72 @@
|
||||
### Install the server on `odroid`
|
||||
|
||||
```sh
|
||||
curl -sfL https://get.k3s.io | sh -s - server \
|
||||
--node-ip="100.70.193.97" \
|
||||
--node-external-ip="100.70.193.97" \
|
||||
--tls-san="100.70.193.97" \
|
||||
--flannel-iface="tailscale0" \
|
||||
--flannel-external-ip \
|
||||
--flannel-backend="vxlan"
|
||||
```
|
||||
|
||||
Get the join token from `odroid`:
|
||||
|
||||
sudo systemctl is-active k3s
|
||||
sudo cat /var/lib/rancher/k3s/server/node-token
|
||||
|
||||
Copy that value into `K3S_TOKEN` on each node only when running its join
|
||||
command. Do not commit the token to this file.
|
||||
|
||||
### Join `rudolf-T`
|
||||
|
||||
Run on `rudolf-T`:
|
||||
|
||||
```sh
|
||||
export K3S_TOKEN='<TODO>'
|
||||
curl -sfL https://get.k3s.io | \
|
||||
K3S_URL="https://100.70.193.97:6443" \
|
||||
K3S_TOKEN="$K3S_TOKEN" \
|
||||
sh -s - agent \
|
||||
--node-ip="100.105.201.15" \
|
||||
--node-external-ip="100.105.201.15" \
|
||||
--flannel-iface="tailscale0"
|
||||
```
|
||||
|
||||
### Join `xu4`
|
||||
|
||||
Run on `xu4`:
|
||||
|
||||
```sh
|
||||
export K3S_TOKEN='<TODO>'
|
||||
curl -sfL https://get.k3s.io | \
|
||||
K3S_URL="https://100.70.193.97:6443" \
|
||||
K3S_TOKEN="$K3S_TOKEN" \
|
||||
sh -s - agent \
|
||||
--node-ip="100.96.92.102" \
|
||||
--node-external-ip="100.96.92.102" \
|
||||
--flannel-iface="tailscale0"
|
||||
```
|
||||
|
||||
### Verify from `odroid`
|
||||
|
||||
```sh
|
||||
sudo k3s kubectl get nodes -o wide
|
||||
sudo k3s kubectl get pods -A -o wide
|
||||
sudo k3s kubectl get --raw='/readyz?verbose'
|
||||
sudo k3s kubectl get events -A --sort-by=.lastTimestamp | tail -n 50
|
||||
```
|
||||
|
||||
### Health
|
||||
# API server health
|
||||
kubectl get --raw='/readyz?verbose'
|
||||
kubectl get --raw='/livez?verbose'
|
||||
|
||||
# Node health and scheduling status
|
||||
kubectl get nodes -o wide
|
||||
|
||||
# All system and workload pods
|
||||
kubectl get pods -A -o wide
|
||||
|
||||
# Recent warnings/errors across namespaces
|
||||
kubectl get events -A --sort-by=.lastTimestamp | tail -n 50
|
||||
@@ -0,0 +1,7 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: agent-gateway
|
||||
resources:
|
||||
- namespace.yaml
|
||||
- rbac.yaml
|
||||
- ../network-policies
|
||||
@@ -0,0 +1,4 @@
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: agent-gateway
|
||||
@@ -0,0 +1,18 @@
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: agent-readonly
|
||||
namespace: agent-gateway
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: agent-readonly-view
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: view
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: agent-readonly
|
||||
namespace: agent-gateway
|
||||
@@ -0,0 +1,21 @@
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: allow-cluster-dns
|
||||
spec:
|
||||
podSelector: {}
|
||||
policyTypes:
|
||||
- Egress
|
||||
egress:
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: kube-system
|
||||
podSelector:
|
||||
matchLabels:
|
||||
k8s-app: kube-dns
|
||||
ports:
|
||||
- protocol: UDP
|
||||
port: 53
|
||||
- protocol: TCP
|
||||
port: 53
|
||||
@@ -0,0 +1,17 @@
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: default-deny-ingress
|
||||
spec:
|
||||
podSelector: {}
|
||||
policyTypes:
|
||||
- Ingress
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: default-deny-egress
|
||||
spec:
|
||||
podSelector: {}
|
||||
policyTypes:
|
||||
- Egress
|
||||
@@ -0,0 +1,5 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- default-deny.yaml
|
||||
- allow-dns.yaml
|
||||
@@ -0,0 +1,4 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- infrastructure/agent-gateway
|
||||
+50
-10
@@ -1,11 +1,10 @@
|
||||
|
||||
# Gitea Actions stack. Uses a dedicated act_runner cache-server so that
|
||||
# job-container networks can reach the cache backend via Swarm service DNS
|
||||
# (gitea_cache_server:37051) instead of the unreachable IP the built-in
|
||||
# cache server binds to per-job.
|
||||
services:
|
||||
server:
|
||||
image: docker.gitea.com/gitea:1.27.0
|
||||
image: docker.gitea.com/gitea:1.27.3
|
||||
# environment:
|
||||
# - USER_UID=1000
|
||||
# - USER_GID=1000
|
||||
@@ -14,9 +13,13 @@ services:
|
||||
- gitea
|
||||
- nginx_ingress
|
||||
volumes:
|
||||
- data:/data
|
||||
- /root/volumes/gitea:/data
|
||||
- /etc/timezone:/etc/timezone:ro
|
||||
- /etc/localtime:/etc/localtime:ro
|
||||
deploy:
|
||||
placement:
|
||||
constraints:
|
||||
- node.hostname == odroid
|
||||
ports:
|
||||
- "3002:3000"
|
||||
- "222:22"
|
||||
@@ -47,6 +50,41 @@ services:
|
||||
secrets:
|
||||
- gitea_cache_secret
|
||||
deploy:
|
||||
placement:
|
||||
constraints:
|
||||
- node.hostname == odroid
|
||||
restart_policy:
|
||||
condition: on-failure
|
||||
delay: 5s
|
||||
max_attempts: 3
|
||||
|
||||
runner_rudolf_t:
|
||||
image: docker.io/gitea/act_runner:nightly
|
||||
entrypoint: ["/usr/local/bin/runner-run.sh"]
|
||||
environment:
|
||||
CONFIG_FILE: /cfg/config.yaml
|
||||
GITEA_INSTANCE_URL: "https://gitea.rudolf.keenetic.link"
|
||||
GITEA_RUNNER_REGISTRATION_TOKEN: "ZIB6L21imA3XJkq65Cq1h43NzB5U1zhAJK9ybfCz"
|
||||
GITEA_RUNNER_NAME: "runner2"
|
||||
GITEA_RUNNER_LABELS: "runner_label1"
|
||||
ACT_RUNNER_EXTERNAL_SECRET_FILE: /run/secrets/gitea_cache_secret
|
||||
networks:
|
||||
- gitea
|
||||
configs:
|
||||
- source: runner_config
|
||||
target: /cfg/config.yaml
|
||||
mode: 0444
|
||||
- source: runner_run
|
||||
target: /usr/local/bin/runner-run.sh
|
||||
mode: 0755
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
secrets:
|
||||
- gitea_cache_secret
|
||||
deploy:
|
||||
placement:
|
||||
constraints:
|
||||
- node.hostname == rudolf-T
|
||||
restart_policy:
|
||||
condition: on-failure
|
||||
delay: 5s
|
||||
@@ -82,6 +120,8 @@ networks:
|
||||
external: true
|
||||
gitea:
|
||||
driver: overlay
|
||||
driver_opts:
|
||||
com.docker.network.driver.mtu: "1200"
|
||||
|
||||
configs:
|
||||
runner_config:
|
||||
@@ -99,11 +139,11 @@ secrets:
|
||||
external: true
|
||||
|
||||
volumes:
|
||||
data:
|
||||
driver: local
|
||||
driver_opts:
|
||||
type: nfs
|
||||
o: addr=192.168.1.96
|
||||
device: :/root/volumes/gitea
|
||||
# device: :/media/DATA/runtime/gitea
|
||||
# data:
|
||||
# driver: local
|
||||
# driver_opts:
|
||||
# type: nfs
|
||||
# o: addr=192.168.1.96
|
||||
# device: :/root/volumes/gitea
|
||||
# # device: :/media/DATA/runtime/gitea
|
||||
cache_data:
|
||||
Reference in New Issue
Block a user