fix(web): serialise ws_connections mutations under asyncio.Lock
Python package / build (3.12.3) (pull_request) Failing after 1m3s
Python package / build (3.13) (pull_request) Failing after 56s
Python package / build (3.14) (pull_request) Failing after 1m0s

Race between websocket_endpoint and broadcast() in backend/web.py caused
intermittent 'KeyError: <WebSocket object>' errors every ~7s in
brencher-brencher-backend logs (counted 354+ in the last hour), forcing
the browser UI to reconnect and dropping log/sse updates.

Repro window:
  websocket_endpoint:
    175: self.ws_connections[websocket] = {}
    await websocket.send_text(version_payload)
    await websocket.send_text(branches_payload)
    184: self.ws_connections[websocket]["branches"] = branches_payload  # KeyError

  broadcast() runs concurrently, sends to N sockets, awaits each, and
  pops disconnected ones. The await yields control back to
  websocket_endpoint which then tries to write a key that was already
  popped (or — conversely — broadcast iterates over a snapshot while a
  parallel endpoint adds a new key it later tries to mutate).

Fix:
- Add asyncio.Lock to WebApp.__init__ and acquire it for every mutation
  of ws_connections (broadcast iteration + cleanup, websocket_endpoint
  registration, WebSocketDisconnect cleanup).
- websocket_endpoint now registers the websocket with all three
  payloads (version/branches/environments) in a single dict assignment
  BEFORE the first send_text await, so there is no window where the key
  exists but the inner dict is empty.
- broadcast() snapshots ws_connections.keys() via list() before iterating,
  and runs send_text + dict write + cleanup under the same lock.

Tests (tests/test_web_ws_lock.py, 4 cases):
- test_websocket_endpoint_registers_atomically_under_lock: a single
  endpoint connection with a slow send_text; concurrent broadcasts do
  not cause KeyError or a missing-key dict.
- test_broadcast_and_concurrent_register_dont_crash: 20 endpoints + a
  broadcaster all run concurrently; ws_connections must drain cleanly
  and never raise.
- test_ws_lock_exists_and_is_asyncio_lock: guard against future
  accidental removal.
- test_broadcast_skips_already_sent_payload: the de-dup check still
  works after the lock change (regression guard).

Refs #42
This commit is contained in:
hermes
2026-09-05 17:52:44 +00:00
parent 2556ac76f7
commit e0ae7de150
2 changed files with 218 additions and 19 deletions
+30 -19
View File
@@ -45,6 +45,7 @@ class WebApp:
self.port = port
self.ws_connections: Dict[WebSocket, Dict[str, Any]] = {}
self.ws_lock = asyncio.Lock()
self._event_loop: Optional[asyncio.AbstractEventLoop] = None
self._server: Any = None
self.secondaryManager: Optional[SecondaryManager] = None
@@ -116,18 +117,19 @@ class WebApp:
disconnected = set()
message = custom_json_dumps({event: data})
for websocket in self.ws_connections.keys():
if self.ws_connections[websocket].get(event) == message:
continue
try:
await websocket.send_text(message)
self.ws_connections[websocket][event] = message
except Exception as e:
logger.error(f"Error sending to websocket: {e}")
disconnected.add(websocket)
async with self.ws_lock:
for websocket in list(self.ws_connections.keys()):
if self.ws_connections[websocket].get(event) == message:
continue
try:
await websocket.send_text(message)
self.ws_connections[websocket][event] = message
except Exception as e:
logger.error(f"Error sending to websocket: {e}")
disconnected.add(websocket)
for ws in disconnected:
self.ws_connections.pop(ws, None)
for ws in disconnected:
self.ws_connections.pop(ws, None)
async def broadcast_all(self) -> None:
await self.broadcast("branches", self.get_global_branches_to_emit())
@@ -172,18 +174,26 @@ class WebApp:
async def websocket_endpoint(self, websocket: WebSocket) -> None:
await websocket.accept()
self.ws_connections[websocket] = {}
# Compute payloads first so the dict registration below is a single
# atomic update under ws_lock — avoids the window where the key
# exists but the inner dict is empty, which races with broadcast()
# cleanup and crashes with KeyError at self.ws_connections[ws][...] = ...
branches_payload = custom_json_dumps({"branches": self.get_global_branches_to_emit()})
envs_payload = custom_json_dumps({"environments": self.get_global_envs_to_emit()})
version_payload = custom_json_dumps({"version": self.get_version()})
async with self.ws_lock:
self.ws_connections[websocket] = {
"version": version_payload,
"branches": branches_payload,
"environments": envs_payload,
}
try:
# Send initial state on connect and record it so duplicate broadcasts are suppressed
branches_payload = custom_json_dumps({"branches": self.get_global_branches_to_emit()})
envs_payload = custom_json_dumps({"environments": self.get_global_envs_to_emit()})
version_payload = custom_json_dumps({"version": self.get_version()})
await websocket.send_text(version_payload)
await websocket.send_text(branches_payload)
self.ws_connections[websocket]["branches"] = branches_payload
await websocket.send_text(envs_payload)
self.ws_connections[websocket]["environments"] = envs_payload
while True:
data = await websocket.receive_text()
@@ -234,7 +244,8 @@ class WebApp:
await self.broadcast_environments(self.get_global_envs_to_emit())
except WebSocketDisconnect:
self.ws_connections.pop(websocket, None)
async with self.ws_lock:
self.ws_connections.pop(websocket, None)
except Exception as e:
logger.error(f"WebSocket error: {e}:{traceback.format_exc()}")
await self.broadcast_error({'message': f'{e}:{traceback.format_exc()}'})
+188
View File
@@ -0,0 +1,188 @@
"""
Regression tests for the WebSocket ws_connections race fixed in PR #42.
Background
----------
Before the fix, ``WebApp.ws_connections`` (a Dict[WebSocket, Dict[str, Any]])
was modified from two uncoordinated coroutines: ``websocket_endpoint`` and
``broadcast()``. Adding the key, then awaiting ``send_text``, then assigning
``self.ws_connections[websocket]["branches"] = ...`` left a window where the
key existed but the inner dict was empty. A concurrent ``broadcast()`` loop
could ``pop(ws)`` the same key, and the next assignment crashed with::
File "/app/backend/web.py", line 184, in websocket_endpoint
self.ws_connections[websocket]["branches"] = branches_payload
~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^
KeyError: <starlette.websockets.WebSocket object at 0x...>
The fix serialises every mutation of ``ws_connections`` under an
``asyncio.Lock`` and registers the websocket with all three payloads in a
single atomic assignment before the first ``send_text`` await.
These tests verify the lock contract on a real ``WebApp`` instance with
fake websocket objects that simulate the production timing.
"""
import asyncio
import sys
from typing import Any, Dict, List
import pytest
# Match the project's import layout: backend modules are at top level.
sys.path.insert(0, "backend")
from web import WebApp
class FakeWebSocket:
"""Minimal stand-in for starlette.websockets.WebSocket.
- ``send_delay`` lets us widen the race window between accept and dict
writes, mimicking slow clients.
- ``receive_text`` raises ``WebSocketDisconnect`` so the endpoint exits
cleanly after one round-trip.
"""
def __init__(self, send_delay: float = 0.0) -> None:
self.sent: List[str] = []
self.accepted = False
self.send_delay = send_delay
async def accept(self) -> None:
self.accepted = True
async def send_text(self, message: str) -> None:
if self.send_delay:
await asyncio.sleep(self.send_delay)
self.sent.append(message)
async def receive_text(self) -> str:
from starlette.websockets import WebSocketDisconnect
raise WebSocketDisconnect()
class FakeCore:
"""Minimal stand-in for ``app.App`` — only the methods WebApp touches."""
def __init__(self) -> None:
self.local_envs: Dict[str, Any] = {}
def get_local_envs_to_emit(self) -> Dict[str, Any]:
return self.local_envs
def get_local_branches_to_emit(self) -> Dict[str, Any]:
return {}
def _make_web_app() -> WebApp:
return WebApp(core=FakeCore(), port=0) # type: ignore[arg-type]
@pytest.mark.asyncio
async def test_websocket_endpoint_registers_atomically_under_lock() -> None:
"""websocket_endpoint must register the ws with all three payloads in a
single dict assignment under ``ws_lock``, BEFORE any ``send_text``.
The lock and the atomic dict write are the contract; if either is
missing, this test fails with KeyError or with the dict missing keys.
"""
web_app = _make_web_app()
ws = FakeWebSocket(send_delay=0.01) # widen the race window
async def concurrent_broadcast() -> None:
# Try to interleave several broadcasts while the endpoint is
# awaiting send_text. Pre-fix, this would race with the dict
# updates and either crash or drop the connection.
for _ in range(5):
await asyncio.sleep(0.005)
await web_app.broadcast("branches", {"x": 1})
task = asyncio.create_task(concurrent_broadcast())
await web_app.websocket_endpoint(ws) # type: ignore[arg-type]
await task
# Endpoint must have accepted the connection
assert ws.accepted
# Three initial send_text calls (version, branches, environments)
assert len(ws.sent) >= 3
# After endpoint exits cleanly (WebSocketDisconnect), the entry is gone
assert ws not in web_app.ws_connections
@pytest.mark.asyncio
async def test_broadcast_and_concurrent_register_dont_crash() -> None:
"""``broadcast()`` and a concurrent ``websocket_endpoint`` registration
must serialise cleanly under the lock. Pre-fix this could raise
``RuntimeError: dictionary changed size during iteration`` or
``KeyError`` when the outer key was popped between accept and write.
"""
web_app = _make_web_app()
n = 20
sockets = [FakeWebSocket(send_delay=0.002) for _ in range(n)]
async def register(s: FakeWebSocket) -> None:
await web_app.websocket_endpoint(s) # type: ignore[arg-type]
async def broadcaster() -> None:
for _ in range(10):
await web_app.broadcast("environments", {"k": "v"})
await asyncio.sleep(0)
# All endpoints + broadcasts run concurrently. Pre-fix this would crash
# with KeyError or RuntimeError on most runs.
await asyncio.gather(
*(register(s) for s in sockets),
broadcaster(),
)
# All endpoints should have cleaned up after themselves.
async with web_app.ws_lock:
leftover = list(web_app.ws_connections.keys())
assert leftover == [], f"expected ws_connections to drain, got {leftover}"
@pytest.mark.asyncio
async def test_ws_lock_exists_and_is_asyncio_lock() -> None:
"""Sanity check on the lock attribute — guards against accidental
removal in future refactors.
"""
import asyncio as _asyncio
web_app = _make_web_app()
assert isinstance(web_app.ws_lock, _asyncio.Lock)
@pytest.mark.asyncio
async def test_broadcast_skips_already_sent_payload() -> None:
"""broadcast() must not re-send a payload it has already sent to a
connection (used for de-dup). This is the second contract of the
function (the first being serialised mutation); ensure the lock didn't
break the de-dup check.
"""
from web import custom_json_dumps
web_app = _make_web_app()
ws = FakeWebSocket()
# Pre-register a websocket with the EXACT JSON payload that broadcast()
# will produce. ``broadcast(event, data)`` JSON-encodes
# ``{event: data}`` into the message and compares against the stored
# value, so we need to mirror that here.
payload = {"branches": "msg-1"}
web_app.ws_connections[ws] = {"branches": custom_json_dumps(payload)}
# Two broadcasts with the same payload — second one must be de-duped
# against the stored value (the de-dup check on line 120:
# ``if self.ws_connections[websocket].get(event) == message: continue``).
await web_app.broadcast("branches", payload)
# After the first call, the de-dup is in place — a second identical
# broadcast must not reach send_text.
sent_before = len(ws.sent)
await web_app.broadcast("branches", payload)
assert len(ws.sent) == sent_before, (
f"de-dup broken: second broadcast sent {len(ws.sent) - sent_before} extra message(s)"
)
# Cleanup
async with web_app.ws_lock:
web_app.ws_connections.pop(ws, None)