rework secondaries (#78)

* rework secondaries
This commit is contained in:
2026-05-09 17:31:50 +02:00
committed by GitHub
parent c3e0bff795
commit 234ffdd634
8 changed files with 189 additions and 139 deletions
+28 -1
View File
@@ -1,16 +1,40 @@
# Use official Python image with uv pre-installed
FROM python:3.12.3-alpine
FROM python:3.12.3-slim AS typecheck
# Install uv (if not present)
RUN pip install uv
# Set workdir
WORKDIR /app
# Install development dependencies and run mypy before the runtime image is built.
COPY requirements.txt requirements-dev.txt mypy.ini ./
RUN uv venv .venv && \
uv pip install -r requirements-dev.txt
COPY backend/ ./backend/
COPY tests/ ./tests/
RUN . .venv/bin/activate && \
mypy --no-incremental --config-file mypy.ini backend tests && \
touch /tmp/typecheck.done
# Use official Python image with uv pre-installed
FROM python:3.12.3-alpine AS base
# Install uv (if not present)
RUN pip install uv
# Install git
RUN apk update && apk upgrade
RUN apk add --no-cache git docker-cli
# Set workdir
WORKDIR /app
# Copy requirements and install dependencies with uv
COPY requirements.txt ./
RUN uv venv .venv && \
uv pip install -r requirements.txt
@@ -21,6 +45,9 @@ COPY frontend/ ./frontend/
# Expose port for the web server
EXPOSE 5001
# Ensure the mypy pre-build stage is executed.
COPY --from=typecheck /tmp/typecheck.done /tmp/typecheck.done
# Set environment variables
ENV PYTHONUNBUFFERED=1
ENV PATH="/app/.venv/bin:$PATH"
+39 -50
View File
@@ -16,7 +16,7 @@ from fastapi.responses import FileResponse
from enironment import Environment, wrap_in_cached, SharedStateHolder, SharedStateConflictError, SharedState, get_step
from processing import reset_caches
from slave import SlaveConnector
from secondary import SecondaryManager
from steps.git import GitClone
from steps.step import CachingStep
@@ -67,8 +67,6 @@ ws_connections: Dict[WebSocket, Dict[str, Any]] = {}
# Event loop reference for thread-safe async calls
_event_loop: Optional[asyncio.AbstractEventLoop] = None
# Slave connection
slave_connectors: List[SlaveConnector] = []
environment_update_event = threading.Event()
@@ -168,12 +166,12 @@ def get_global_envs_to_emit() -> Any:
global environments
local_envs = get_local_envs_to_emit()
merged: Dict[str, Any] = local_envs
for connector in slave_connectors:
slave_envs = connector.environments
common_keys = set(merged.keys()) & set(slave_envs.keys())
for connector in secondaryManager or []:
v = connector.environments
common_keys = set(merged.keys()) & set(v.keys())
if len(common_keys) > 0:
_schedule_async(broadcast_error({'message': f"Conflict: both master and slave have environment with id {common_keys}"}))
merged = merge_dicts(merged, slave_envs)
_schedule_async(broadcast_error({'message': f"Conflict: both master and secondary have environment with id {common_keys}"}))
merged = merge_dicts(merged, v)
return merged
@@ -197,7 +195,7 @@ def get_local_branches_to_emit() -> Dict[str, Dict[str, List[Any]]]:
def get_global_branches_to_emit() -> Dict[str, Dict[str, List[Any]]]:
local_branches: Dict[str, Dict[str, List[Any]]] = get_local_branches_to_emit()
merged: Dict[str, Dict[str, List[Any]]] = local_branches
for connector in slave_connectors:
for connector in secondaryManager or []:
merged = merge_dicts(merged, connector.branches)
return merged
@@ -230,6 +228,10 @@ async def broadcast(event: str, data: Any) -> None:
for ws in disconnected:
ws_connections.pop(ws, None)
async def broadcast_all() -> None:
await broadcast("branches", get_global_branches_to_emit())
await broadcast("environments", get_global_envs_to_emit())
async def broadcast_branches(data: Any) -> None:
await broadcast("branches", data)
@@ -266,47 +268,42 @@ async def websocket_endpoint(websocket: WebSocket) -> None:
if "update" in message:
update_data = message.get("update") or {}
logger.info(f"Received environment update: {update_data}")
if update_data.get('id') == '':
if secondaryManager:
await secondaryManager.send({"update": update_data})
id = update_data.get('id', '')
if id == '':
reset_caches(list(environments.values()))
else:
env = environments[update_data.get('id', '')]
branches_update: List[Tuple[str, str]] | None = update_data.get('branches')
elif id not in environments.keys() and id not in { j for it in secondaryManager or [] for j in it.environments.keys()}:
logger.warning(f"Received update for unknown environment id {id}")
continue
elif id in environments.keys():
env = environments.get(id, None)
expected_token = update_data.get('token', '')
if branches_update:
if 'branches' in update_data:
if not env:
raise RuntimeError(f"Unknown env {update_data.get('id', '')}")
try:
env.state.set_branches(branches_update, expected_token=expected_token)
except SharedStateConflictError as conflict:
await broadcast_error({
"code": "BRANCH_STATE_CONFLICT",
"message": "Branch state changed, refresh and retry",
"envId": env.id,
})
continue
env.state.set_branches(update_data.get('branches', []), expected_token=expected_token)
if 'dry' in update_data:
if not env:
raise RuntimeError(f"Unknown env {update_data.get('id', '')}")
env.state.set_dry(bool(update_data['dry']), expected_token)
p = get_step(env.pipeline, type(env.state))
if isinstance(p, CachingStep):
p.reset()
logger.info(f"Updated environment {env.id} branches to {update_data.get('branches')}, dry={update_data.get('dry')}")
environment_update_event.set()
if slave_connectors:
for connector in slave_connectors:
await connector.send({"update": update_data})
if env:
p = get_step(env.pipeline, type(env.state))
if isinstance(p, CachingStep):
p.reset()
logger.info(f"Updated environment {env.id} branches to {update_data.get('branches')}, dry={update_data.get('dry')}")
environment_update_event.set()
await broadcast_environments(get_global_envs_to_emit())
except WebSocketDisconnect as e:
ws_connections.pop(websocket, None)
except Exception as e:
logger.error(f"WebSocket error: {e}")
await broadcast_error({'message': 'WebSocket exception'})
ws_connections.pop(websocket, None)
logger.error(f"WebSocket error: {e}:{traceback.format_exc()}")
await broadcast_error({'message': f'{e}:{traceback.format_exc()}'})
# await websocket.close()
# ws_connections.pop(websocket, None)
# --- SIGCHLD Handler ---
@@ -324,24 +321,16 @@ def sigchld_handler(signum: int, frame: Any) -> None:
# --- Startup Event ---
secondaryManager: Optional[SecondaryManager] = None
@app.on_event("startup")
async def startup_event() -> None:
global _event_loop, slave_connectors
global _event_loop, secondaryManager
_event_loop = asyncio.get_event_loop()
seen_urls: set[str] = set()
for env in environments.values():
for url in env.slave_urls:
if url in seen_urls:
continue
seen_urls.add(url)
connector = SlaveConnector(
url,
on_branches=lambda: broadcast_branches(get_global_branches_to_emit()),
on_environments=lambda: broadcast_environments(get_global_envs_to_emit()),
on_error=broadcast_error,
)
connector.start()
slave_connectors.append(connector)
secondaryManager = SecondaryManager(
urls = os.getenv("SECONDARY_BRENCHER", ""),
on_update=lambda: broadcast_all(),
)
# --- App Class (used by CLI and integration tests) ---
-1
View File
@@ -101,5 +101,4 @@ brencher2 = Environment(
checkPing,
logUrls
],
slave_urls=["http://host.docker.internal:5002"],
)
+1 -2
View File
@@ -65,6 +65,5 @@ brencher_local2 = Environment(
dockerSwarmCheck,
deployDocker,
unmerge
],
slave_urls=["http://host.docker.internal:5002"],
]
)
+1 -2
View File
@@ -2,7 +2,7 @@ from __future__ import annotations
import logging
from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from dataclasses import dataclass
from typing import List, runtime_checkable, Protocol, Tuple, TypeAlias, Any
from typing import TypeVar
@@ -16,7 +16,6 @@ class Environment:
id: str
state: SharedStateHolder
pipeline: List[AbstractStep]
slave_urls: List[str] = field(default_factory=list)
def __post_init__(self) -> None:
for p in self.pipeline:
+119
View File
@@ -0,0 +1,119 @@
import asyncio
import json
import logging
from typing import Any, Awaitable, Callable, Dict, List, Optional, Iterator
import websockets
logger = logging.getLogger(__name__)
class SecondaryConnector:
def __init__(
self,
url: str,
on_update: Callable[[], Awaitable[None]],
) -> None:
self._url = url
self.branches: Dict[str, Any] = {}
self.environments: Dict[str, Any] = {}
self._task: Optional[asyncio.Task[None]] = None
self._on_update = on_update
self._ws: Optional[Any] = None
def start(self) -> None:
self._task = asyncio.create_task(self._connect())
@property
def is_active(self) -> bool:
return self._task is not None and not self._task.done()
async def send(self, msg: Dict[str, Any]) -> None:
payload = json.dumps(msg)
max_attempts = 3
timeout_seconds = 10.0
for attempt in range(1, max_attempts + 1):
if self._ws:
try:
await asyncio.wait_for(self._ws.send(payload), timeout=timeout_seconds)
return
except Exception as e:
logger.warning(f"Send attempt {attempt}/{max_attempts} failed for {self._url}: {e}")
if attempt < max_attempts:
await asyncio.sleep(timeout_seconds * attempt)
logger.error(f"Dropping message for secondary {self._url} after {max_attempts} attempts")
async def _connect(self) -> None:
logger.info(f"Connecting to secondary at {self._url}")
reconnect_delay_seconds = 5
while True:
try:
ws_url = (
self._url
.replace('http://', 'ws://')
.replace('https://', 'wss://')
+ '/ws'
)
async with websockets.connect(
ws_url,
ping_interval=20,
ping_timeout=20,
close_timeout=5,
) as ws:
self._ws = ws
logger.info(f"Connected to secondary WebSocket at {self._url}")
reconnect_delay_seconds = 5
async for msg in ws:
parsed = json.loads(msg)
if "branches" in parsed:
self.branches = parsed["branches"]
await self._on_update()
elif "environments" in parsed:
self.environments = parsed["environments"]
await self._on_update()
elif "error" in parsed:
await self._on_update()
except asyncio.CancelledError:
raise
except Exception:
logger.exception(
f"Unexpected secondary connector error for {self._url}. "
f"Retrying in {reconnect_delay_seconds}s"
)
finally:
self._ws = None
await asyncio.sleep(reconnect_delay_seconds)
reconnect_delay_seconds = min(reconnect_delay_seconds * 2, 60)
class SecondaryManager():
def __init__(self,
urls: str,
on_update: Callable[[], Awaitable[None]],
) -> None:
self.lst: List[SecondaryConnector] = []
urls_list = [it.strip() for it in urls.split(",") if it.strip()]
logger.info(f"SECONDARY_BRENCHER is {urls_list}")
for url in urls_list:
connector = SecondaryConnector(
url,
on_update=on_update,
)
connector.start()
self.lst.append(connector)
def __iter__(self) -> Iterator[SecondaryConnector]:
return iter(self.lst)
async def send(self, msg: Dict[str, Any]) -> None:
await asyncio.gather(*(connector.send(msg) for connector in self.lst), return_exceptions=True)
-82
View File
@@ -1,82 +0,0 @@
import asyncio
import json
import logging
from typing import Any, Awaitable, Callable, Dict, Optional
import websockets
logger = logging.getLogger(__name__)
class SlaveConnector:
"""Manages the WebSocket connection to a slave brencher instance."""
def __init__(
self,
slave_url: str,
on_branches: Callable[[], Awaitable[None]],
on_environments: Callable[[], Awaitable[None]],
on_error: Callable[[Any], Awaitable[None]],
) -> None:
self._slave_url = slave_url
self.branches: Dict[str, Any] = {}
self.environments: Dict[str, Any] = {}
self._send_queue: asyncio.Queue[Dict[str, Any]] = asyncio.Queue()
self._task: Optional[asyncio.Task[None]] = None
self._on_branches = on_branches
self._on_environments = on_environments
self._on_error = on_error
def start(self) -> None:
"""Start the background task that maintains the slave WebSocket connection."""
self._task = asyncio.create_task(self._connect())
@property
def is_active(self) -> bool:
return self._task is not None and not self._task.done()
async def send(self, msg: Dict[str, Any]) -> None:
"""Enqueue a message to be forwarded to the slave."""
await self._send_queue.put(msg)
async def _connect(self) -> None:
"""Continuously connect (and reconnect) to the slave WebSocket."""
logger.info(f"SLAVE_BRENCHER set, will connect to slave at {self._slave_url}")
while True:
try:
ws_url = (
self._slave_url
.replace('http://', 'ws://')
.replace('https://', 'wss://')
+ '/ws'
)
async with websockets.connect(ws_url) as ws:
logger.info("Connected to slave WebSocket")
async def handle_messages() -> None:
async for msg in ws:
parsed = json.loads(msg)
if "branches" in parsed:
self.branches = parsed["branches"]
await self._on_branches()
elif "environments" in parsed:
self.environments = parsed["environments"]
await self._on_environments()
elif "error" in parsed:
await self._on_error(parsed["error"])
async def handle_sends() -> None:
while True:
msg = await self._send_queue.get()
await ws.send(json.dumps(msg))
await asyncio.gather(
handle_messages(),
handle_sends(),
)
except Exception as e:
logger.error(f"Could not connect to SLAVE_BRENCHER {self._slave_url}: {e}")
await asyncio.sleep(60)
+1 -1
View File
@@ -17,7 +17,7 @@ services:
- VIRTUAL_HOST=brencher.rudolf.keenetic.link
- LETSENCRYPT_HOST=brencher.rudolf.keenetic.link
- PROFILES=brencher2,torrserv_proxy,immich,registry,gmail_mcp
# - SLAVE_BRENCHER='http://100.70.193.97:5002' .venv/bin/python3 backend/app.py brencher_localX
- SECONDARY_BRENCHER=http://host.docker.internal:5002
deploy:
replicas: 1
restart_policy: