diff --git a/Dockerfile b/Dockerfile index 0fbf3b5..3320786 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,16 +1,40 @@ # Use official Python image with uv pre-installed -FROM python:3.12.3-alpine +FROM python:3.12.3-slim AS typecheck + +# Install uv (if not present) +RUN pip install uv + +# Set workdir +WORKDIR /app + +# Install development dependencies and run mypy before the runtime image is built. +COPY requirements.txt requirements-dev.txt mypy.ini ./ + +RUN uv venv .venv && \ + uv pip install -r requirements-dev.txt + +COPY backend/ ./backend/ +COPY tests/ ./tests/ + +RUN . .venv/bin/activate && \ + mypy --no-incremental --config-file mypy.ini backend tests && \ + touch /tmp/typecheck.done + +# Use official Python image with uv pre-installed +FROM python:3.12.3-alpine AS base # Install uv (if not present) RUN pip install uv # Install git RUN apk update && apk upgrade RUN apk add --no-cache git docker-cli + # Set workdir WORKDIR /app # Copy requirements and install dependencies with uv COPY requirements.txt ./ + RUN uv venv .venv && \ uv pip install -r requirements.txt @@ -21,6 +45,9 @@ COPY frontend/ ./frontend/ # Expose port for the web server EXPOSE 5001 +# Ensure the mypy pre-build stage is executed. +COPY --from=typecheck /tmp/typecheck.done /tmp/typecheck.done + # Set environment variables ENV PYTHONUNBUFFERED=1 ENV PATH="/app/.venv/bin:$PATH" diff --git a/backend/app.py b/backend/app.py index 04ab75e..7831b4c 100644 --- a/backend/app.py +++ b/backend/app.py @@ -16,7 +16,7 @@ from fastapi.responses import FileResponse from enironment import Environment, wrap_in_cached, SharedStateHolder, SharedStateConflictError, SharedState, get_step from processing import reset_caches -from slave import SlaveConnector +from secondary import SecondaryManager from steps.git import GitClone from steps.step import CachingStep @@ -67,8 +67,6 @@ ws_connections: Dict[WebSocket, Dict[str, Any]] = {} # Event loop reference for thread-safe async calls _event_loop: Optional[asyncio.AbstractEventLoop] = None -# Slave connection -slave_connectors: List[SlaveConnector] = [] environment_update_event = threading.Event() @@ -168,12 +166,12 @@ def get_global_envs_to_emit() -> Any: global environments local_envs = get_local_envs_to_emit() merged: Dict[str, Any] = local_envs - for connector in slave_connectors: - slave_envs = connector.environments - common_keys = set(merged.keys()) & set(slave_envs.keys()) + for connector in secondaryManager or []: + v = connector.environments + common_keys = set(merged.keys()) & set(v.keys()) if len(common_keys) > 0: - _schedule_async(broadcast_error({'message': f"Conflict: both master and slave have environment with id {common_keys}"})) - merged = merge_dicts(merged, slave_envs) + _schedule_async(broadcast_error({'message': f"Conflict: both master and secondary have environment with id {common_keys}"})) + merged = merge_dicts(merged, v) return merged @@ -197,7 +195,7 @@ def get_local_branches_to_emit() -> Dict[str, Dict[str, List[Any]]]: def get_global_branches_to_emit() -> Dict[str, Dict[str, List[Any]]]: local_branches: Dict[str, Dict[str, List[Any]]] = get_local_branches_to_emit() merged: Dict[str, Dict[str, List[Any]]] = local_branches - for connector in slave_connectors: + for connector in secondaryManager or []: merged = merge_dicts(merged, connector.branches) return merged @@ -230,6 +228,10 @@ async def broadcast(event: str, data: Any) -> None: for ws in disconnected: ws_connections.pop(ws, None) +async def broadcast_all() -> None: + await broadcast("branches", get_global_branches_to_emit()) + await broadcast("environments", get_global_envs_to_emit()) + async def broadcast_branches(data: Any) -> None: await broadcast("branches", data) @@ -266,47 +268,42 @@ async def websocket_endpoint(websocket: WebSocket) -> None: if "update" in message: update_data = message.get("update") or {} logger.info(f"Received environment update: {update_data}") - - if update_data.get('id') == '': + if secondaryManager: + await secondaryManager.send({"update": update_data}) + id = update_data.get('id', '') + if id == '': reset_caches(list(environments.values())) - else: - env = environments[update_data.get('id', '')] - branches_update: List[Tuple[str, str]] | None = update_data.get('branches') + elif id not in environments.keys() and id not in { j for it in secondaryManager or [] for j in it.environments.keys()}: + logger.warning(f"Received update for unknown environment id {id}") + continue + elif id in environments.keys(): + env = environments.get(id, None) expected_token = update_data.get('token', '') - if branches_update: + if 'branches' in update_data: if not env: raise RuntimeError(f"Unknown env {update_data.get('id', '')}") - try: - env.state.set_branches(branches_update, expected_token=expected_token) - except SharedStateConflictError as conflict: - await broadcast_error({ - "code": "BRANCH_STATE_CONFLICT", - "message": "Branch state changed, refresh and retry", - "envId": env.id, - }) - continue + env.state.set_branches(update_data.get('branches', []), expected_token=expected_token) if 'dry' in update_data: if not env: raise RuntimeError(f"Unknown env {update_data.get('id', '')}") env.state.set_dry(bool(update_data['dry']), expected_token) - p = get_step(env.pipeline, type(env.state)) - if isinstance(p, CachingStep): - p.reset() - logger.info(f"Updated environment {env.id} branches to {update_data.get('branches')}, dry={update_data.get('dry')}") - environment_update_event.set() - if slave_connectors: - for connector in slave_connectors: - await connector.send({"update": update_data}) + if env: + p = get_step(env.pipeline, type(env.state)) + if isinstance(p, CachingStep): + p.reset() + logger.info(f"Updated environment {env.id} branches to {update_data.get('branches')}, dry={update_data.get('dry')}") + environment_update_event.set() await broadcast_environments(get_global_envs_to_emit()) except WebSocketDisconnect as e: ws_connections.pop(websocket, None) except Exception as e: - logger.error(f"WebSocket error: {e}") - await broadcast_error({'message': 'WebSocket exception'}) - ws_connections.pop(websocket, None) + logger.error(f"WebSocket error: {e}:{traceback.format_exc()}") + await broadcast_error({'message': f'{e}:{traceback.format_exc()}'}) + # await websocket.close() + # ws_connections.pop(websocket, None) # --- SIGCHLD Handler --- @@ -324,24 +321,16 @@ def sigchld_handler(signum: int, frame: Any) -> None: # --- Startup Event --- +secondaryManager: Optional[SecondaryManager] = None + @app.on_event("startup") async def startup_event() -> None: - global _event_loop, slave_connectors + global _event_loop, secondaryManager _event_loop = asyncio.get_event_loop() - seen_urls: set[str] = set() - for env in environments.values(): - for url in env.slave_urls: - if url in seen_urls: - continue - seen_urls.add(url) - connector = SlaveConnector( - url, - on_branches=lambda: broadcast_branches(get_global_branches_to_emit()), - on_environments=lambda: broadcast_environments(get_global_envs_to_emit()), - on_error=broadcast_error, - ) - connector.start() - slave_connectors.append(connector) + secondaryManager = SecondaryManager( + urls = os.getenv("SECONDARY_BRENCHER", ""), + on_update=lambda: broadcast_all(), + ) # --- App Class (used by CLI and integration tests) --- diff --git a/backend/configs/brencher2.py b/backend/configs/brencher2.py index e349532..4f90cdc 100644 --- a/backend/configs/brencher2.py +++ b/backend/configs/brencher2.py @@ -101,5 +101,4 @@ brencher2 = Environment( checkPing, logUrls ], - slave_urls=["http://host.docker.internal:5002"], ) diff --git a/backend/configs/brencher_local2.py b/backend/configs/brencher_local2.py index 3fca6e5..b665307 100644 --- a/backend/configs/brencher_local2.py +++ b/backend/configs/brencher_local2.py @@ -65,6 +65,5 @@ brencher_local2 = Environment( dockerSwarmCheck, deployDocker, unmerge - ], - slave_urls=["http://host.docker.internal:5002"], + ] ) diff --git a/backend/enironment.py b/backend/enironment.py index fceb07c..87692a7 100644 --- a/backend/enironment.py +++ b/backend/enironment.py @@ -2,7 +2,7 @@ from __future__ import annotations import logging from abc import ABC, abstractmethod -from dataclasses import dataclass, field +from dataclasses import dataclass from typing import List, runtime_checkable, Protocol, Tuple, TypeAlias, Any from typing import TypeVar @@ -16,7 +16,6 @@ class Environment: id: str state: SharedStateHolder pipeline: List[AbstractStep] - slave_urls: List[str] = field(default_factory=list) def __post_init__(self) -> None: for p in self.pipeline: diff --git a/backend/secondary.py b/backend/secondary.py new file mode 100644 index 0000000..17e7a07 --- /dev/null +++ b/backend/secondary.py @@ -0,0 +1,119 @@ +import asyncio +import json +import logging +from typing import Any, Awaitable, Callable, Dict, List, Optional, Iterator + +import websockets + +logger = logging.getLogger(__name__) + + +class SecondaryConnector: + + def __init__( + self, + url: str, + on_update: Callable[[], Awaitable[None]], + ) -> None: + self._url = url + self.branches: Dict[str, Any] = {} + self.environments: Dict[str, Any] = {} + self._task: Optional[asyncio.Task[None]] = None + self._on_update = on_update + self._ws: Optional[Any] = None + + def start(self) -> None: + self._task = asyncio.create_task(self._connect()) + + @property + def is_active(self) -> bool: + return self._task is not None and not self._task.done() + + async def send(self, msg: Dict[str, Any]) -> None: + + payload = json.dumps(msg) + max_attempts = 3 + timeout_seconds = 10.0 + + for attempt in range(1, max_attempts + 1): + if self._ws: + try: + await asyncio.wait_for(self._ws.send(payload), timeout=timeout_seconds) + return + except Exception as e: + logger.warning(f"Send attempt {attempt}/{max_attempts} failed for {self._url}: {e}") + + if attempt < max_attempts: + await asyncio.sleep(timeout_seconds * attempt) + + logger.error(f"Dropping message for secondary {self._url} after {max_attempts} attempts") + + async def _connect(self) -> None: + logger.info(f"Connecting to secondary at {self._url}") + reconnect_delay_seconds = 5 + + while True: + try: + ws_url = ( + self._url + .replace('http://', 'ws://') + .replace('https://', 'wss://') + + '/ws' + ) + + async with websockets.connect( + ws_url, + ping_interval=20, + ping_timeout=20, + close_timeout=5, + ) as ws: + self._ws = ws + logger.info(f"Connected to secondary WebSocket at {self._url}") + reconnect_delay_seconds = 5 + + async for msg in ws: + parsed = json.loads(msg) + if "branches" in parsed: + self.branches = parsed["branches"] + await self._on_update() + elif "environments" in parsed: + self.environments = parsed["environments"] + await self._on_update() + elif "error" in parsed: + await self._on_update() + + + except asyncio.CancelledError: + raise + except Exception: + logger.exception( + f"Unexpected secondary connector error for {self._url}. " + f"Retrying in {reconnect_delay_seconds}s" + ) + finally: + self._ws = None + + await asyncio.sleep(reconnect_delay_seconds) + reconnect_delay_seconds = min(reconnect_delay_seconds * 2, 60) + +class SecondaryManager(): + def __init__(self, + urls: str, + on_update: Callable[[], Awaitable[None]], + ) -> None: + self.lst: List[SecondaryConnector] = [] + urls_list = [it.strip() for it in urls.split(",") if it.strip()] + logger.info(f"SECONDARY_BRENCHER is {urls_list}") + for url in urls_list: + connector = SecondaryConnector( + url, + on_update=on_update, + ) + connector.start() + self.lst.append(connector) + + def __iter__(self) -> Iterator[SecondaryConnector]: + return iter(self.lst) + + async def send(self, msg: Dict[str, Any]) -> None: + await asyncio.gather(*(connector.send(msg) for connector in self.lst), return_exceptions=True) diff --git a/backend/slave.py b/backend/slave.py deleted file mode 100644 index aa00cea..0000000 --- a/backend/slave.py +++ /dev/null @@ -1,82 +0,0 @@ -import asyncio -import json -import logging -from typing import Any, Awaitable, Callable, Dict, Optional - -import websockets - -logger = logging.getLogger(__name__) - - -class SlaveConnector: - """Manages the WebSocket connection to a slave brencher instance.""" - - def __init__( - self, - slave_url: str, - on_branches: Callable[[], Awaitable[None]], - on_environments: Callable[[], Awaitable[None]], - on_error: Callable[[Any], Awaitable[None]], - ) -> None: - self._slave_url = slave_url - self.branches: Dict[str, Any] = {} - self.environments: Dict[str, Any] = {} - self._send_queue: asyncio.Queue[Dict[str, Any]] = asyncio.Queue() - self._task: Optional[asyncio.Task[None]] = None - self._on_branches = on_branches - self._on_environments = on_environments - self._on_error = on_error - - def start(self) -> None: - """Start the background task that maintains the slave WebSocket connection.""" - self._task = asyncio.create_task(self._connect()) - - @property - def is_active(self) -> bool: - return self._task is not None and not self._task.done() - - async def send(self, msg: Dict[str, Any]) -> None: - """Enqueue a message to be forwarded to the slave.""" - await self._send_queue.put(msg) - - async def _connect(self) -> None: - """Continuously connect (and reconnect) to the slave WebSocket.""" - logger.info(f"SLAVE_BRENCHER set, will connect to slave at {self._slave_url}") - - while True: - try: - ws_url = ( - self._slave_url - .replace('http://', 'ws://') - .replace('https://', 'wss://') - + '/ws' - ) - - async with websockets.connect(ws_url) as ws: - logger.info("Connected to slave WebSocket") - - async def handle_messages() -> None: - async for msg in ws: - parsed = json.loads(msg) - if "branches" in parsed: - self.branches = parsed["branches"] - await self._on_branches() - elif "environments" in parsed: - self.environments = parsed["environments"] - await self._on_environments() - elif "error" in parsed: - await self._on_error(parsed["error"]) - - async def handle_sends() -> None: - while True: - msg = await self._send_queue.get() - await ws.send(json.dumps(msg)) - - await asyncio.gather( - handle_messages(), - handle_sends(), - ) - - except Exception as e: - logger.error(f"Could not connect to SLAVE_BRENCHER {self._slave_url}: {e}") - await asyncio.sleep(60) diff --git a/docker-compose.yml b/docker-compose.yml index 3b93baf..5f1ed92 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,7 +17,7 @@ services: - VIRTUAL_HOST=brencher.rudolf.keenetic.link - LETSENCRYPT_HOST=brencher.rudolf.keenetic.link - PROFILES=brencher2,torrserv_proxy,immich,registry,gmail_mcp - # - SLAVE_BRENCHER='http://100.70.193.97:5002' .venv/bin/python3 backend/app.py brencher_localX + - SECONDARY_BRENCHER=http://host.docker.internal:5002 deploy: replicas: 1 restart_policy: