fix(web): serialise ws_connections mutations under asyncio.Lock
Race between websocket_endpoint and broadcast() in backend/web.py caused
intermittent 'KeyError: <WebSocket object>' errors every ~7s in
brencher-brencher-backend logs (counted 354+ in the last hour), forcing
the browser UI to reconnect and dropping log/sse updates.
Repro window:
websocket_endpoint:
175: self.ws_connections[websocket] = {}
await websocket.send_text(version_payload)
await websocket.send_text(branches_payload)
184: self.ws_connections[websocket]["branches"] = branches_payload # KeyError
broadcast() runs concurrently, sends to N sockets, awaits each, and
pops disconnected ones. The await yields control back to
websocket_endpoint which then tries to write a key that was already
popped (or — conversely — broadcast iterates over a snapshot while a
parallel endpoint adds a new key it later tries to mutate).
Fix:
- Add asyncio.Lock to WebApp.__init__ and acquire it for every mutation
of ws_connections (broadcast iteration + cleanup, websocket_endpoint
registration, WebSocketDisconnect cleanup).
- websocket_endpoint now registers the websocket with all three
payloads (version/branches/environments) in a single dict assignment
BEFORE the first send_text await, so there is no window where the key
exists but the inner dict is empty.
- broadcast() snapshots ws_connections.keys() via list() before iterating,
and runs send_text + dict write + cleanup under the same lock.
Tests (tests/test_web_ws_lock.py, 4 cases):
- test_websocket_endpoint_registers_atomically_under_lock: a single
endpoint connection with a slow send_text; concurrent broadcasts do
not cause KeyError or a missing-key dict.
- test_broadcast_and_concurrent_register_dont_crash: 20 endpoints + a
broadcaster all run concurrently; ws_connections must drain cleanly
and never raise.
- test_ws_lock_exists_and_is_asyncio_lock: guard against future
accidental removal.
- test_broadcast_skips_already_sent_payload: the de-dup check still
works after the lock change (regression guard).
Refs #42
This commit is contained in:
+30
-19
@@ -45,6 +45,7 @@ class WebApp:
|
||||
self.port = port
|
||||
|
||||
self.ws_connections: Dict[WebSocket, Dict[str, Any]] = {}
|
||||
self.ws_lock = asyncio.Lock()
|
||||
self._event_loop: Optional[asyncio.AbstractEventLoop] = None
|
||||
self._server: Any = None
|
||||
self.secondaryManager: Optional[SecondaryManager] = None
|
||||
@@ -116,18 +117,19 @@ class WebApp:
|
||||
disconnected = set()
|
||||
message = custom_json_dumps({event: data})
|
||||
|
||||
for websocket in self.ws_connections.keys():
|
||||
if self.ws_connections[websocket].get(event) == message:
|
||||
continue
|
||||
try:
|
||||
await websocket.send_text(message)
|
||||
self.ws_connections[websocket][event] = message
|
||||
except Exception as e:
|
||||
logger.error(f"Error sending to websocket: {e}")
|
||||
disconnected.add(websocket)
|
||||
async with self.ws_lock:
|
||||
for websocket in list(self.ws_connections.keys()):
|
||||
if self.ws_connections[websocket].get(event) == message:
|
||||
continue
|
||||
try:
|
||||
await websocket.send_text(message)
|
||||
self.ws_connections[websocket][event] = message
|
||||
except Exception as e:
|
||||
logger.error(f"Error sending to websocket: {e}")
|
||||
disconnected.add(websocket)
|
||||
|
||||
for ws in disconnected:
|
||||
self.ws_connections.pop(ws, None)
|
||||
for ws in disconnected:
|
||||
self.ws_connections.pop(ws, None)
|
||||
|
||||
async def broadcast_all(self) -> None:
|
||||
await self.broadcast("branches", self.get_global_branches_to_emit())
|
||||
@@ -172,18 +174,26 @@ class WebApp:
|
||||
|
||||
async def websocket_endpoint(self, websocket: WebSocket) -> None:
|
||||
await websocket.accept()
|
||||
self.ws_connections[websocket] = {}
|
||||
|
||||
# Compute payloads first so the dict registration below is a single
|
||||
# atomic update under ws_lock — avoids the window where the key
|
||||
# exists but the inner dict is empty, which races with broadcast()
|
||||
# cleanup and crashes with KeyError at self.ws_connections[ws][...] = ...
|
||||
branches_payload = custom_json_dumps({"branches": self.get_global_branches_to_emit()})
|
||||
envs_payload = custom_json_dumps({"environments": self.get_global_envs_to_emit()})
|
||||
version_payload = custom_json_dumps({"version": self.get_version()})
|
||||
|
||||
async with self.ws_lock:
|
||||
self.ws_connections[websocket] = {
|
||||
"version": version_payload,
|
||||
"branches": branches_payload,
|
||||
"environments": envs_payload,
|
||||
}
|
||||
|
||||
try:
|
||||
# Send initial state on connect and record it so duplicate broadcasts are suppressed
|
||||
branches_payload = custom_json_dumps({"branches": self.get_global_branches_to_emit()})
|
||||
envs_payload = custom_json_dumps({"environments": self.get_global_envs_to_emit()})
|
||||
version_payload = custom_json_dumps({"version": self.get_version()})
|
||||
await websocket.send_text(version_payload)
|
||||
await websocket.send_text(branches_payload)
|
||||
self.ws_connections[websocket]["branches"] = branches_payload
|
||||
await websocket.send_text(envs_payload)
|
||||
self.ws_connections[websocket]["environments"] = envs_payload
|
||||
|
||||
while True:
|
||||
data = await websocket.receive_text()
|
||||
@@ -234,7 +244,8 @@ class WebApp:
|
||||
await self.broadcast_environments(self.get_global_envs_to_emit())
|
||||
|
||||
except WebSocketDisconnect:
|
||||
self.ws_connections.pop(websocket, None)
|
||||
async with self.ws_lock:
|
||||
self.ws_connections.pop(websocket, None)
|
||||
except Exception as e:
|
||||
logger.error(f"WebSocket error: {e}:{traceback.format_exc()}")
|
||||
await self.broadcast_error({'message': f'{e}:{traceback.format_exc()}'})
|
||||
|
||||
@@ -0,0 +1,188 @@
|
||||
"""
|
||||
Regression tests for the WebSocket ws_connections race fixed in PR #42.
|
||||
|
||||
Background
|
||||
----------
|
||||
Before the fix, ``WebApp.ws_connections`` (a Dict[WebSocket, Dict[str, Any]])
|
||||
was modified from two uncoordinated coroutines: ``websocket_endpoint`` and
|
||||
``broadcast()``. Adding the key, then awaiting ``send_text``, then assigning
|
||||
``self.ws_connections[websocket]["branches"] = ...`` left a window where the
|
||||
key existed but the inner dict was empty. A concurrent ``broadcast()`` loop
|
||||
could ``pop(ws)`` the same key, and the next assignment crashed with::
|
||||
|
||||
File "/app/backend/web.py", line 184, in websocket_endpoint
|
||||
self.ws_connections[websocket]["branches"] = branches_payload
|
||||
~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^
|
||||
KeyError: <starlette.websockets.WebSocket object at 0x...>
|
||||
|
||||
The fix serialises every mutation of ``ws_connections`` under an
|
||||
``asyncio.Lock`` and registers the websocket with all three payloads in a
|
||||
single atomic assignment before the first ``send_text`` await.
|
||||
|
||||
These tests verify the lock contract on a real ``WebApp`` instance with
|
||||
fake websocket objects that simulate the production timing.
|
||||
"""
|
||||
import asyncio
|
||||
import sys
|
||||
from typing import Any, Dict, List
|
||||
|
||||
import pytest
|
||||
|
||||
# Match the project's import layout: backend modules are at top level.
|
||||
sys.path.insert(0, "backend")
|
||||
|
||||
from web import WebApp
|
||||
|
||||
|
||||
class FakeWebSocket:
|
||||
"""Minimal stand-in for starlette.websockets.WebSocket.
|
||||
|
||||
- ``send_delay`` lets us widen the race window between accept and dict
|
||||
writes, mimicking slow clients.
|
||||
- ``receive_text`` raises ``WebSocketDisconnect`` so the endpoint exits
|
||||
cleanly after one round-trip.
|
||||
"""
|
||||
|
||||
def __init__(self, send_delay: float = 0.0) -> None:
|
||||
self.sent: List[str] = []
|
||||
self.accepted = False
|
||||
self.send_delay = send_delay
|
||||
|
||||
async def accept(self) -> None:
|
||||
self.accepted = True
|
||||
|
||||
async def send_text(self, message: str) -> None:
|
||||
if self.send_delay:
|
||||
await asyncio.sleep(self.send_delay)
|
||||
self.sent.append(message)
|
||||
|
||||
async def receive_text(self) -> str:
|
||||
from starlette.websockets import WebSocketDisconnect
|
||||
raise WebSocketDisconnect()
|
||||
|
||||
|
||||
class FakeCore:
|
||||
"""Minimal stand-in for ``app.App`` — only the methods WebApp touches."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.local_envs: Dict[str, Any] = {}
|
||||
|
||||
def get_local_envs_to_emit(self) -> Dict[str, Any]:
|
||||
return self.local_envs
|
||||
|
||||
def get_local_branches_to_emit(self) -> Dict[str, Any]:
|
||||
return {}
|
||||
|
||||
|
||||
def _make_web_app() -> WebApp:
|
||||
return WebApp(core=FakeCore(), port=0) # type: ignore[arg-type]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_websocket_endpoint_registers_atomically_under_lock() -> None:
|
||||
"""websocket_endpoint must register the ws with all three payloads in a
|
||||
single dict assignment under ``ws_lock``, BEFORE any ``send_text``.
|
||||
|
||||
The lock and the atomic dict write are the contract; if either is
|
||||
missing, this test fails with KeyError or with the dict missing keys.
|
||||
"""
|
||||
web_app = _make_web_app()
|
||||
ws = FakeWebSocket(send_delay=0.01) # widen the race window
|
||||
|
||||
async def concurrent_broadcast() -> None:
|
||||
# Try to interleave several broadcasts while the endpoint is
|
||||
# awaiting send_text. Pre-fix, this would race with the dict
|
||||
# updates and either crash or drop the connection.
|
||||
for _ in range(5):
|
||||
await asyncio.sleep(0.005)
|
||||
await web_app.broadcast("branches", {"x": 1})
|
||||
|
||||
task = asyncio.create_task(concurrent_broadcast())
|
||||
await web_app.websocket_endpoint(ws) # type: ignore[arg-type]
|
||||
await task
|
||||
|
||||
# Endpoint must have accepted the connection
|
||||
assert ws.accepted
|
||||
# Three initial send_text calls (version, branches, environments)
|
||||
assert len(ws.sent) >= 3
|
||||
# After endpoint exits cleanly (WebSocketDisconnect), the entry is gone
|
||||
assert ws not in web_app.ws_connections
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_broadcast_and_concurrent_register_dont_crash() -> None:
|
||||
"""``broadcast()`` and a concurrent ``websocket_endpoint`` registration
|
||||
must serialise cleanly under the lock. Pre-fix this could raise
|
||||
``RuntimeError: dictionary changed size during iteration`` or
|
||||
``KeyError`` when the outer key was popped between accept and write.
|
||||
"""
|
||||
web_app = _make_web_app()
|
||||
|
||||
n = 20
|
||||
sockets = [FakeWebSocket(send_delay=0.002) for _ in range(n)]
|
||||
|
||||
async def register(s: FakeWebSocket) -> None:
|
||||
await web_app.websocket_endpoint(s) # type: ignore[arg-type]
|
||||
|
||||
async def broadcaster() -> None:
|
||||
for _ in range(10):
|
||||
await web_app.broadcast("environments", {"k": "v"})
|
||||
await asyncio.sleep(0)
|
||||
|
||||
# All endpoints + broadcasts run concurrently. Pre-fix this would crash
|
||||
# with KeyError or RuntimeError on most runs.
|
||||
await asyncio.gather(
|
||||
*(register(s) for s in sockets),
|
||||
broadcaster(),
|
||||
)
|
||||
|
||||
# All endpoints should have cleaned up after themselves.
|
||||
async with web_app.ws_lock:
|
||||
leftover = list(web_app.ws_connections.keys())
|
||||
assert leftover == [], f"expected ws_connections to drain, got {leftover}"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ws_lock_exists_and_is_asyncio_lock() -> None:
|
||||
"""Sanity check on the lock attribute — guards against accidental
|
||||
removal in future refactors.
|
||||
"""
|
||||
import asyncio as _asyncio
|
||||
web_app = _make_web_app()
|
||||
assert isinstance(web_app.ws_lock, _asyncio.Lock)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_broadcast_skips_already_sent_payload() -> None:
|
||||
"""broadcast() must not re-send a payload it has already sent to a
|
||||
connection (used for de-dup). This is the second contract of the
|
||||
function (the first being serialised mutation); ensure the lock didn't
|
||||
break the de-dup check.
|
||||
"""
|
||||
from web import custom_json_dumps
|
||||
web_app = _make_web_app()
|
||||
ws = FakeWebSocket()
|
||||
|
||||
# Pre-register a websocket with the EXACT JSON payload that broadcast()
|
||||
# will produce. ``broadcast(event, data)`` JSON-encodes
|
||||
# ``{event: data}`` into the message and compares against the stored
|
||||
# value, so we need to mirror that here.
|
||||
payload = {"branches": "msg-1"}
|
||||
web_app.ws_connections[ws] = {"branches": custom_json_dumps(payload)}
|
||||
|
||||
# Two broadcasts with the same payload — second one must be de-duped
|
||||
# against the stored value (the de-dup check on line 120:
|
||||
# ``if self.ws_connections[websocket].get(event) == message: continue``).
|
||||
await web_app.broadcast("branches", payload)
|
||||
# After the first call, the de-dup is in place — a second identical
|
||||
# broadcast must not reach send_text.
|
||||
sent_before = len(ws.sent)
|
||||
await web_app.broadcast("branches", payload)
|
||||
|
||||
assert len(ws.sent) == sent_before, (
|
||||
f"de-dup broken: second broadcast sent {len(ws.sent) - sent_before} extra message(s)"
|
||||
)
|
||||
|
||||
# Cleanup
|
||||
async with web_app.ws_lock:
|
||||
web_app.ws_connections.pop(ws, None)
|
||||
Reference in New Issue
Block a user