Move slave connectivity to separate class with multi-connector support (#76)
* Initial plan * Move slave connectivity to separate SlaveConnector class in backend/slave.py Agent-Logs-Url: https://github.com/rudolf1/brencher/sessions/8dab3353-0172-49d4-818c-cff27d5b13b5 Co-authored-by: rudolf1 <5525769+rudolf1@users.noreply.github.com> * Support multiple slave connectors via Environment.slave_urls list Agent-Logs-Url: https://github.com/rudolf1/brencher/sessions/081649bc-80b1-407d-9895-cdd621727907 Co-authored-by: rudolf1 <5525769+rudolf1@users.noreply.github.com> * Add slave_urls to brencher2/brencher_local2 configs; remove SLAVE_BRENCHER from docker-compose.yml Agent-Logs-Url: https://github.com/rudolf1/brencher/sessions/6f458415-0672-4842-9e0c-8670daf77283 Co-authored-by: rudolf1 <5525769+rudolf1@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: rudolf1 <5525769+rudolf1@users.noreply.github.com>
This commit is contained in:
+34
-66
@@ -9,7 +9,6 @@ import traceback
|
||||
from dataclasses import asdict, replace
|
||||
from typing import Any, Dict, List, Optional, TypeVar, Tuple
|
||||
|
||||
import websockets
|
||||
from dotenv import load_dotenv
|
||||
from fastapi import FastAPI, Response, WebSocket, WebSocketDisconnect
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
@@ -17,6 +16,7 @@ from fastapi.responses import FileResponse
|
||||
|
||||
from enironment import Environment, wrap_in_cached, SharedStateHolder, SharedStateConflictError, SharedState, get_step
|
||||
from processing import reset_caches
|
||||
from slave import SlaveConnector
|
||||
from steps.git import GitClone
|
||||
from steps.step import CachingStep
|
||||
|
||||
@@ -59,8 +59,6 @@ FRONTEND_DIR = os.path.join(os.path.dirname(__file__), '../frontend')
|
||||
|
||||
# In-memory state
|
||||
environments: Dict[str, Environment] = {}
|
||||
environments_slaves: Dict[str, Any] = {}
|
||||
branches_slaves: Dict[str, Dict[str, Any]] = {}
|
||||
state_lock = threading.Lock()
|
||||
|
||||
# Active WebSocket connections
|
||||
@@ -69,10 +67,8 @@ ws_connections: Dict[WebSocket, Dict[str, Any]] = {}
|
||||
# Event loop reference for thread-safe async calls
|
||||
_event_loop: Optional[asyncio.AbstractEventLoop] = None
|
||||
|
||||
# Slave connection state
|
||||
slave_ws_task: Optional[asyncio.Task[None]] = None
|
||||
slave_send_queue: asyncio.Queue[Dict[str, Any]] = asyncio.Queue()
|
||||
slave_url = os.getenv('SLAVE_BRENCHER')
|
||||
# Slave connection
|
||||
slave_connectors: List[SlaveConnector] = []
|
||||
|
||||
environment_update_event = threading.Event()
|
||||
|
||||
@@ -169,13 +165,16 @@ def get_local_envs_to_emit() -> Dict[str, Dict[str, Any]]:
|
||||
|
||||
|
||||
def get_global_envs_to_emit() -> Any:
|
||||
global environments, environments_slaves
|
||||
global environments
|
||||
local_envs = get_local_envs_to_emit()
|
||||
merge_result = merge_dicts(local_envs, environments_slaves)
|
||||
common_keys = set(local_envs.keys()) & set(environments_slaves.keys())
|
||||
if len(common_keys) > 0:
|
||||
_schedule_async(broadcast_error({'message': f"Conflict: both master and slave have environment with id {common_keys}"}))
|
||||
return merge_result
|
||||
merged: Dict[str, Any] = local_envs
|
||||
for connector in slave_connectors:
|
||||
slave_envs = connector.environments
|
||||
common_keys = set(merged.keys()) & set(slave_envs.keys())
|
||||
if len(common_keys) > 0:
|
||||
_schedule_async(broadcast_error({'message': f"Conflict: both master and slave have environment with id {common_keys}"}))
|
||||
merged = merge_dicts(merged, slave_envs)
|
||||
return merged
|
||||
|
||||
|
||||
|
||||
@@ -196,9 +195,11 @@ def get_local_branches_to_emit() -> Dict[str, Dict[str, List[Any]]]:
|
||||
|
||||
|
||||
def get_global_branches_to_emit() -> Dict[str, Dict[str, List[Any]]]:
|
||||
global branches_slaves
|
||||
local_branches: Dict[str, Dict[str, List[Any]]] = get_local_branches_to_emit()
|
||||
return merge_dicts(local_branches, branches_slaves)
|
||||
merged: Dict[str, Dict[str, List[Any]]] = local_branches
|
||||
for connector in slave_connectors:
|
||||
merged = merge_dicts(merged, connector.branches)
|
||||
return merged
|
||||
|
||||
|
||||
# --- Async Helper for Calling from Sync Threads ---
|
||||
@@ -294,8 +295,9 @@ async def websocket_endpoint(websocket: WebSocket) -> None:
|
||||
logger.info(f"Updated environment {env.id} branches to {update_data.get('branches')}, dry={update_data.get('dry')}")
|
||||
environment_update_event.set()
|
||||
|
||||
if slave_ws_task is not None:
|
||||
await slave_send_queue.put({"update": update_data})
|
||||
if slave_connectors:
|
||||
for connector in slave_connectors:
|
||||
await connector.send({"update": update_data})
|
||||
|
||||
await broadcast_environments(get_global_envs_to_emit())
|
||||
|
||||
@@ -307,52 +309,6 @@ async def websocket_endpoint(websocket: WebSocket) -> None:
|
||||
ws_connections.pop(websocket, None)
|
||||
|
||||
|
||||
# --- Slave Connection ---
|
||||
|
||||
async def connect_to_slave() -> None:
|
||||
"""Connect to a slave brencher instance via a single WebSocket."""
|
||||
global branches_slaves, environments_slaves
|
||||
|
||||
if not slave_url:
|
||||
return
|
||||
|
||||
logger.info(f"SLAVE_BRENCHER set, will connect to slave at {slave_url}")
|
||||
|
||||
while True:
|
||||
try:
|
||||
ws_url = slave_url.replace('http://', 'ws://').replace('https://', 'wss://') + '/ws'
|
||||
|
||||
async with websockets.connect(ws_url) as ws:
|
||||
logger.info("Connected to slave WebSocket")
|
||||
|
||||
async def handle_messages() -> None:
|
||||
global branches_slaves, environments_slaves
|
||||
async for msg in ws:
|
||||
parsed = json.loads(msg)
|
||||
if "branches" in parsed:
|
||||
branches_slaves = parsed["branches"]
|
||||
await broadcast_branches(get_global_branches_to_emit())
|
||||
elif "environments" in parsed:
|
||||
environments_slaves = parsed["environments"]
|
||||
await broadcast_environments(get_global_envs_to_emit())
|
||||
elif "error" in parsed:
|
||||
await broadcast_error(parsed["error"])
|
||||
|
||||
async def handle_sends() -> None:
|
||||
while True:
|
||||
msg = await slave_send_queue.get()
|
||||
await ws.send(json.dumps(msg))
|
||||
|
||||
await asyncio.gather(
|
||||
handle_messages(),
|
||||
handle_sends(),
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Could not connect to SLAVE_BRENCHER {slave_url}: {e}")
|
||||
await asyncio.sleep(60)
|
||||
|
||||
|
||||
# --- SIGCHLD Handler ---
|
||||
|
||||
def sigchld_handler(signum: int, frame: Any) -> None:
|
||||
@@ -370,10 +326,22 @@ def sigchld_handler(signum: int, frame: Any) -> None:
|
||||
|
||||
@app.on_event("startup")
|
||||
async def startup_event() -> None:
|
||||
global _event_loop, slave_ws_task
|
||||
global _event_loop, slave_connectors
|
||||
_event_loop = asyncio.get_event_loop()
|
||||
if slave_url:
|
||||
slave_ws_task = asyncio.create_task(connect_to_slave())
|
||||
seen_urls: set[str] = set()
|
||||
for env in environments.values():
|
||||
for url in env.slave_urls:
|
||||
if url in seen_urls:
|
||||
continue
|
||||
seen_urls.add(url)
|
||||
connector = SlaveConnector(
|
||||
url,
|
||||
on_branches=lambda: broadcast_branches(get_global_branches_to_emit()),
|
||||
on_environments=lambda: broadcast_environments(get_global_envs_to_emit()),
|
||||
on_error=broadcast_error,
|
||||
)
|
||||
connector.start()
|
||||
slave_connectors.append(connector)
|
||||
|
||||
|
||||
# --- App Class (used by CLI and integration tests) ---
|
||||
|
||||
@@ -100,5 +100,6 @@ brencher2 = Environment(
|
||||
deployDocker,
|
||||
checkPing,
|
||||
logUrls
|
||||
]
|
||||
],
|
||||
slave_urls=["http://host.docker.internal:5002"],
|
||||
)
|
||||
|
||||
@@ -65,5 +65,6 @@ brencher_local2 = Environment(
|
||||
dockerSwarmCheck,
|
||||
deployDocker,
|
||||
unmerge
|
||||
]
|
||||
],
|
||||
slave_urls=["http://host.docker.internal:5002"],
|
||||
)
|
||||
|
||||
@@ -2,7 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, field
|
||||
from typing import List, runtime_checkable, Protocol, Tuple, TypeAlias, Any
|
||||
from typing import TypeVar
|
||||
|
||||
@@ -16,6 +16,7 @@ class Environment:
|
||||
id: str
|
||||
state: SharedStateHolder
|
||||
pipeline: List[AbstractStep]
|
||||
slave_urls: List[str] = field(default_factory=list)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
for p in self.pipeline:
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from typing import Any, Awaitable, Callable, Dict, Optional
|
||||
|
||||
import websockets
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SlaveConnector:
|
||||
"""Manages the WebSocket connection to a slave brencher instance."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
slave_url: str,
|
||||
on_branches: Callable[[], Awaitable[None]],
|
||||
on_environments: Callable[[], Awaitable[None]],
|
||||
on_error: Callable[[Any], Awaitable[None]],
|
||||
) -> None:
|
||||
self._slave_url = slave_url
|
||||
self.branches: Dict[str, Any] = {}
|
||||
self.environments: Dict[str, Any] = {}
|
||||
self._send_queue: asyncio.Queue[Dict[str, Any]] = asyncio.Queue()
|
||||
self._task: Optional[asyncio.Task[None]] = None
|
||||
self._on_branches = on_branches
|
||||
self._on_environments = on_environments
|
||||
self._on_error = on_error
|
||||
|
||||
def start(self) -> None:
|
||||
"""Start the background task that maintains the slave WebSocket connection."""
|
||||
self._task = asyncio.create_task(self._connect())
|
||||
|
||||
@property
|
||||
def is_active(self) -> bool:
|
||||
return self._task is not None and not self._task.done()
|
||||
|
||||
async def send(self, msg: Dict[str, Any]) -> None:
|
||||
"""Enqueue a message to be forwarded to the slave."""
|
||||
await self._send_queue.put(msg)
|
||||
|
||||
async def _connect(self) -> None:
|
||||
"""Continuously connect (and reconnect) to the slave WebSocket."""
|
||||
logger.info(f"SLAVE_BRENCHER set, will connect to slave at {self._slave_url}")
|
||||
|
||||
while True:
|
||||
try:
|
||||
ws_url = (
|
||||
self._slave_url
|
||||
.replace('http://', 'ws://')
|
||||
.replace('https://', 'wss://')
|
||||
+ '/ws'
|
||||
)
|
||||
|
||||
async with websockets.connect(ws_url) as ws:
|
||||
logger.info("Connected to slave WebSocket")
|
||||
|
||||
async def handle_messages() -> None:
|
||||
async for msg in ws:
|
||||
parsed = json.loads(msg)
|
||||
if "branches" in parsed:
|
||||
self.branches = parsed["branches"]
|
||||
await self._on_branches()
|
||||
elif "environments" in parsed:
|
||||
self.environments = parsed["environments"]
|
||||
await self._on_environments()
|
||||
elif "error" in parsed:
|
||||
await self._on_error(parsed["error"])
|
||||
|
||||
async def handle_sends() -> None:
|
||||
while True:
|
||||
msg = await self._send_queue.get()
|
||||
await ws.send(json.dumps(msg))
|
||||
|
||||
await asyncio.gather(
|
||||
handle_messages(),
|
||||
handle_sends(),
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Could not connect to SLAVE_BRENCHER {self._slave_url}: {e}")
|
||||
await asyncio.sleep(60)
|
||||
@@ -17,7 +17,6 @@ services:
|
||||
- VIRTUAL_HOST=brencher.rudolf.keenetic.link
|
||||
- LETSENCRYPT_HOST=brencher.rudolf.keenetic.link
|
||||
- PROFILES=brencher2,torrserv_proxy,immich,registry,gmail_mcp
|
||||
- SLAVE_BRENCHER=http://host.docker.internal:5002
|
||||
# - SLAVE_BRENCHER='http://100.70.193.97:5002' .venv/bin/python3 backend/app.py brencher_localX
|
||||
deploy:
|
||||
replicas: 1
|
||||
|
||||
Reference in New Issue
Block a user