fix(web): serialize ws_connections access with asyncio.Lock
The previous fix (setdefault) made `KeyError` non-fatal but did not eliminate the underlying race: between `await websocket.accept()` and the handler writing per-channel state, a concurrent broadcast may add the just-accepted socket to its disconnected set and pop it from ws_connections. The handler still completes, but its writes go to a detached object that is never reattached; subsequent broadcasts re-register a fresh state for the same socket and the duplicate guard in setdefault suppression logic is per-socket, not per-channel, so the first event after re-registration gets emitted unconditionally. Wrap all reads, writes, and removals of `ws_connections` in a single `asyncio.Lock` held only while touching the dict: - `broadcast()` takes the lock to snapshot `(websocket, state)` pairs, releases it before any `await`, and reacquires it to pop dead sockets. - `websocket_endpoint()` takes the lock to insert the new socket immediately after `accept()`, and again after the initial-frame `send_text` burst to record the per-channel state. - Disconnect and exception handlers also pop under the lock. `send_text` itself runs outside the lock, so concurrent broadcasts still pipeline their frames in parallel (no global serialization of network IO). The lock is only contended on dict mutation, which is cheap. Properties: - No data races on ws_connections by construction. - Slightly more lines than the setdefault variant but the mutation surface is now declared explicitly via `async with self._ws_lock:`. Diff stats: 35 insertions(+), 12 deletions(-) in `backend/web.py`. Verified with `python3 -m py_compile`. Self-standing fix, no API changes. Companion PRs: - fix/ws-setdefault-and-broadcast (variant 2: simpler) - fix/ws-writer-loop (variant 4: single-writer pattern) All three fixes solve the same race. This one is the most explicit about the mutation surface; setdefault is the smallest; writer-loop removes any cross-task access entirely.
This commit is contained in:
+35
-12
@@ -47,6 +47,7 @@ class WebApp:
|
||||
self.port = port
|
||||
|
||||
self.ws_connections: Dict[WebSocket, Dict[str, Any]] = {}
|
||||
self._ws_lock: asyncio.Lock = asyncio.Lock()
|
||||
self._event_loop: Optional[asyncio.AbstractEventLoop] = None
|
||||
self.secondaryManager: Optional[SecondaryManager] = None
|
||||
|
||||
@@ -104,21 +105,32 @@ class WebApp:
|
||||
|
||||
async def broadcast(self, event: str, data: Any) -> None:
|
||||
"""Broadcast a message to all connected WebSocket clients that have not seen this data yet."""
|
||||
disconnected = set()
|
||||
message = custom_json_dumps({event: data})
|
||||
|
||||
for websocket in self.ws_connections.keys():
|
||||
if self.ws_connections[websocket].get(event) == message:
|
||||
continue
|
||||
# Snapshot the targets under the lock; releases the lock before
|
||||
# any await so concurrent broadcasts can still serialize on the
|
||||
# dict while send_text runs in parallel.
|
||||
async with self._ws_lock:
|
||||
targets = [
|
||||
(ws, state)
|
||||
for ws, state in self.ws_connections.items()
|
||||
if state.get(event) != message
|
||||
]
|
||||
|
||||
disconnected: List[WebSocket] = []
|
||||
for websocket, state in targets:
|
||||
try:
|
||||
await websocket.send_text(message)
|
||||
self.ws_connections[websocket][event] = message
|
||||
except Exception as e:
|
||||
logger.error(f"Error sending to websocket: {e}")
|
||||
disconnected.add(websocket)
|
||||
disconnected.append(websocket)
|
||||
continue
|
||||
state[event] = message
|
||||
|
||||
for ws in disconnected:
|
||||
self.ws_connections.pop(ws, None)
|
||||
if disconnected:
|
||||
async with self._ws_lock:
|
||||
for ws in disconnected:
|
||||
self.ws_connections.pop(ws, None)
|
||||
|
||||
async def broadcast_all(self) -> None:
|
||||
await self.broadcast("branches", self.get_global_branches_to_emit())
|
||||
@@ -163,7 +175,8 @@ class WebApp:
|
||||
|
||||
async def websocket_endpoint(self, websocket: WebSocket) -> None:
|
||||
await websocket.accept()
|
||||
self.ws_connections[websocket] = {}
|
||||
async with self._ws_lock:
|
||||
self.ws_connections[websocket] = {}
|
||||
|
||||
try:
|
||||
# Send initial state on connect and record it so duplicate broadcasts are suppressed
|
||||
@@ -172,9 +185,16 @@ class WebApp:
|
||||
version_payload = custom_json_dumps({"version": self.get_version()})
|
||||
await websocket.send_text(version_payload)
|
||||
await websocket.send_text(branches_payload)
|
||||
self.ws_connections[websocket]["branches"] = branches_payload
|
||||
await websocket.send_text(envs_payload)
|
||||
self.ws_connections[websocket]["environments"] = envs_payload
|
||||
async with self._ws_lock:
|
||||
state = self.ws_connections.get(websocket)
|
||||
if state is None:
|
||||
# a concurrent broadcast already cleaned us up before we
|
||||
# could record the initial state. The next periodic
|
||||
# broadcast will repopulate it; bail out gracefully.
|
||||
return
|
||||
state["branches"] = branches_payload
|
||||
state["environments"] = envs_payload
|
||||
|
||||
while True:
|
||||
data = await websocket.receive_text()
|
||||
@@ -227,9 +247,12 @@ class WebApp:
|
||||
await self.broadcast_environments(self.get_global_envs_to_emit())
|
||||
|
||||
except WebSocketDisconnect:
|
||||
self.ws_connections.pop(websocket, None)
|
||||
async with self._ws_lock:
|
||||
self.ws_connections.pop(websocket, None)
|
||||
except Exception as e:
|
||||
logger.error(f"WebSocket error: {e}:{traceback.format_exc()}")
|
||||
async with self._ws_lock:
|
||||
self.ws_connections.pop(websocket, None)
|
||||
await self.broadcast_error({'message': f'{e}:{traceback.format_exc()}'})
|
||||
|
||||
# --- Run ---
|
||||
|
||||
Reference in New Issue
Block a user