refactor app concurrency to asyncio taskgroup

Agent-Logs-Url: https://github.com/rudolf1/brencher/sessions/9e719b58-ad2a-4eca-99bf-43c750f71a72

Co-authored-by: rudolf1 <5525769+rudolf1@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2026-05-19 19:34:46 +00:00
committed by GitHub
co-authored by rudolf1
parent efe49119a3
commit 6f3dd6ebd8
5 changed files with 219 additions and 97 deletions
+69 -21
View File
@@ -1,8 +1,8 @@
import asyncio
import logging
import os
import signal
import sys
import threading
import traceback
from typing import Any, Callable, Dict, List, Optional
@@ -27,9 +27,12 @@ class App:
def __init__(self, environments: Dict[str, Environment]) -> None:
self.environments: Dict[str, Environment] = {id: wrap_in_cached(e) for id, e in environments.items()}
self.state_lock = threading.Lock()
self.environment_update_event = threading.Event()
self.environment_update_event = asyncio.Event()
self.environment_update_version = 0
self.emit_callback: Optional[Callable[[], None]] = None
self.reset_requested = False
self.shutdown_event = asyncio.Event()
self.web_app: Any = None
def get_local_envs_to_emit(self) -> Dict[str, Dict[str, Any]]:
env_dtos: Dict[str, Dict[str, Any]] = {}
@@ -88,35 +91,80 @@ class App:
return branches
def processing_thread(self) -> None:
while True:
async def processing_loop(self) -> None:
while not self.shutdown_event.is_set():
import processing
emit = self.emit_callback or (lambda: None)
update_version = self.environment_update_version
with self.state_lock:
logger.info(f"Processing")
if processing.process_all_jobs(list(self.environments.values()), emit):
self.environment_update_event.wait(timeout=1 * 5)
else:
self.environment_update_event.wait(timeout=1 * 60)
self.environment_update_event.clear()
logger.info("Processing")
if self.reset_requested:
await asyncio.to_thread(processing.reset_caches, list(self.environments.values()))
self.reset_requested = False
has_error = await asyncio.to_thread(processing.process_all_jobs, list(self.environments.values()), emit)
if self.shutdown_event.is_set():
return
if self.environment_update_version != update_version:
continue
timeout = 5 if has_error else 60
try:
await asyncio.wait_for(self.environment_update_event.wait(), timeout=timeout)
except TimeoutError:
pass
self.environment_update_event.clear()
def run(self) -> None:
processing = threading.Thread(target=self.processing_thread)
processing.daemon = True
processing.start()
def notify_environment_update(self) -> None:
self.environment_update_version += 1
self.environment_update_event.set()
def request_reset(self) -> None:
self.reset_requested = True
def stop(self) -> None:
self.shutdown_event.set()
self.notify_environment_update()
if self.web_app is not None:
self.web_app.stop()
async def runAsync(self) -> None:
await self.processing_loop()
async def runHeadlessAsync(self) -> None:
await self.processing_loop()
def runHeadless(self) -> None:
"""Run the processing loop on the current thread; blocks forever."""
self.processing_thread()
"""Run the processing loop until cancelled; blocks forever."""
asyncio.run(self.runHeadlessAsync())
def runWeb(self, port: int) -> None:
async def _run_web_server(self, web_app: Any) -> None:
await web_app.start_async()
raise _WebServerStopped()
async def runWebAsync(self, port: int) -> None:
import web
web_app = web.WebApp(core=self, port=port)
self.web_app = web_app
self.emit_callback = web_app.emit_envs
self.run()
web_app.start()
try:
async with asyncio.TaskGroup() as task_group:
task_group.create_task(self.processing_loop())
task_group.create_task(self._run_web_server(web_app))
except* _WebServerStopped:
pass
finally:
self.emit_callback = None
self.web_app = None
def runWeb(self, port: int) -> None:
asyncio.run(self.runWebAsync(port))
def run(self) -> None:
asyncio.run(self.runAsync())
class _WebServerStopped(Exception):
pass
if __name__ == '__main__':
cli_env_ids_list = sys.argv[1:]
+36 -14
View File
@@ -11,11 +11,8 @@ from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import FileResponse
from app import App
from enironment import get_step
from utils import custom_json_dumps
from processing import reset_caches
from secondary import SecondaryManager
from steps.step import CachingStep
logger = logging.getLogger(__name__)
@@ -48,11 +45,12 @@ class WebApp:
self.ws_connections: Dict[WebSocket, Dict[str, Any]] = {}
self._event_loop: Optional[asyncio.AbstractEventLoop] = None
self._server: Any = None
self.secondaryManager: Optional[SecondaryManager] = None
@asynccontextmanager
async def lifespan(fastapi_app: FastAPI) -> AsyncIterator[None]:
self._event_loop = asyncio.get_event_loop()
self._event_loop = asyncio.get_running_loop()
self.secondaryManager = SecondaryManager(
urls=os.getenv("SECONDARY_BRENCHER", ""),
on_update=lambda: self.broadcast_all(),
@@ -94,10 +92,19 @@ class WebApp:
merged = _merge_dicts(merged, connector.branches)
return merged
# --- Async scheduling from sync threads ---
# --- Async scheduling from sync callbacks ---
def _schedule_async(self, coro: Any) -> None:
if self._event_loop is not None and self._event_loop.is_running():
if self._event_loop is None or not self._event_loop.is_running():
coro.close()
return
try:
running_loop = asyncio.get_running_loop()
except RuntimeError:
running_loop = None
if running_loop is self._event_loop:
self._event_loop.create_task(coro)
else:
asyncio.run_coroutine_threadsafe(coro, self._event_loop)
# --- Broadcasting ---
@@ -134,7 +141,7 @@ class WebApp:
await self.broadcast("error", data)
def emit_envs(self) -> None:
"""Sync callback used by App's processing thread to push updates to clients."""
"""Sync callback used by App's processing loop to push updates to clients."""
self._schedule_async(self.broadcast_branches(self.get_global_branches_to_emit()))
self._schedule_async(self.broadcast_environments(self.get_global_envs_to_emit()))
@@ -179,7 +186,7 @@ class WebApp:
await self.secondaryManager.send({"update": update_data})
id = update_data.get('id', '')
if id == '':
reset_caches(list(self.core.environments.values()))
self.core.request_reset()
elif id not in self.core.environments.keys() and id not in {j for it in self.secondaryManager or [] for j in it.environments.keys()}:
logger.warning(f"Received update for unknown environment id {id}")
continue
@@ -196,11 +203,9 @@ class WebApp:
env.state.set_dry(bool(update_data['dry']), expected_token)
if env:
p = get_step(env.pipeline, type(env.state))
if isinstance(p, CachingStep):
p.reset()
self.core.request_reset()
logger.info(f"Updated environment {env.id} branches to {update_data.get('branches')}, dry={update_data.get('dry')}")
self.core.environment_update_event.set()
self.core.notify_environment_update()
await self.broadcast_environments(self.get_global_envs_to_emit())
@@ -212,6 +217,23 @@ class WebApp:
# --- Run ---
def start(self) -> None:
async def start_async(self) -> None:
import uvicorn
uvicorn.run(self.app, host='0.0.0.0', port=self.port, log_level='info')
config = uvicorn.Config(self.app, host='0.0.0.0', port=self.port, log_level='info')
self._server = uvicorn.Server(config)
try:
await self._server.serve()
except asyncio.CancelledError:
self._server.should_exit = True
raise
def start(self) -> None:
asyncio.run(self.start_async())
def stop(self) -> None:
if self._server is None:
return
if self._event_loop is not None and self._event_loop.is_running():
self._event_loop.call_soon_threadsafe(setattr, self._server, "should_exit", True)
else:
self._server.should_exit = True
+50
View File
@@ -0,0 +1,50 @@
import asyncio
import sys
import types
import pytest
import processing
from app import App
class TestAppAsyncio:
@pytest.mark.asyncio
async def test_run_web_async_stops_processing_when_server_exits(self, monkeypatch: pytest.MonkeyPatch) -> None:
process_calls = 0
emit_calls = 0
processing_started = asyncio.Event()
loop = asyncio.get_running_loop()
web_app_args: dict[str, object] = {}
def fake_process_all_jobs(environments: object, onupdate: object) -> bool:
nonlocal process_calls
process_calls += 1
assert callable(onupdate)
onupdate()
loop.call_soon_threadsafe(processing_started.set)
return False
class FakeWebApp:
def __init__(self, core: App, port: int) -> None:
web_app_args["core"] = core
web_app_args["port"] = port
def emit_envs(self) -> None:
nonlocal emit_calls
emit_calls += 1
async def start_async(self) -> None:
await asyncio.wait_for(processing_started.wait(), timeout=1)
monkeypatch.setattr(processing, "process_all_jobs", fake_process_all_jobs)
monkeypatch.setitem(sys.modules, "web", types.SimpleNamespace(WebApp=FakeWebApp))
app = App({})
await asyncio.wait_for(app.runWebAsync(5001), timeout=1)
assert process_calls >= 1
assert emit_calls >= 1
assert web_app_args == {"core": app, "port": 5001}
assert app.emit_callback is None
+20 -21
View File
@@ -1,7 +1,7 @@
import asyncio
import json
import logging
import threading
from contextlib import suppress
from typing import Any, Generator
import docker
@@ -36,30 +36,29 @@ class TestDockerContainer:
logger.info(f"Starting")
app = App({"brencher_local1": brencher_local1.brencher_local1})
# app.runWeb(5001)
processing = threading.Thread(target=lambda: app.runWeb(5001), daemon=True)
processing.start()
processing = asyncio.create_task(app.runWebAsync(5001))
try:
eventually(
lambda: assert_equal(requests.get("http://localhost:5001/state", timeout=5000).status_code, 200, "Server did not respond with 200 OK within timeout"),
20.0,
1.0,
)
# await asyncio.sleep(5000)
state_data = requests.get("http://localhost:5001/state", timeout=5000).json()
logger.info(f"Application state: {json.dumps(state_data, indent=2)}")
eventually(
lambda: assert_equal(requests.get("http://localhost:5001/state", timeout=5000).status_code, 200, "Server did not respond with 200 OK within timeout"),
20.0,
1.0,
)
branches_data = requests.get("http://localhost:5001/branches", timeout=5000).json()
logger.info(f"Application branches: {json.dumps(branches_data, indent=2)}")
assert "brencher_local1" in state_data, "Missing 'brencher_local1' field in response"
assert len(state_data["brencher_local1"]["pipeline"]) > 0, "Steps defined for 'brencher_local1' are empty"
state_data = requests.get("http://localhost:5001/state", timeout=5000).json()
logger.info(f"Application state: {json.dumps(state_data, indent=2)}")
branches_data = requests.get("http://localhost:5001/branches", timeout=5000).json()
logger.info(f"Application branches: {json.dumps(branches_data, indent=2)}")
assert "brencher_local1" in state_data, "Missing 'brencher_local1' field in response"
assert len(state_data["brencher_local1"]["pipeline"]) > 0, "Steps defined for 'brencher_local1' are empty"
assert "brencher_local1" in branches_data, "Missing 'brencher_local1' field in response"
assert len(branches_data["brencher_local1"]) > 0, "No branches found for 'brencher_local1'"
assert "brencher_local1" in branches_data, "Missing 'brencher_local1' field in response"
assert len(branches_data["brencher_local1"]) > 0, "No branches found for 'brencher_local1'"
finally:
processing.cancel()
with suppress(asyncio.CancelledError):
await processing
# TODO Verify DockerContainerDeploy step status is running and has correct image and ports
# {
+44 -41
View File
@@ -43,50 +43,53 @@ class TestDryRunPlaywright:
app = App({env.id: env})
server_thread = threading.Thread(target=lambda: app.runWeb(APP_PORT), daemon=True)
server_thread.start()
try:
# Wait for server to be up
eventually(lambda: assert_equal(requests.get(f"{APP_URL}/state", timeout=5).status_code, 200, "Server did not respond with 200 OK within timeout"))
logger.info("Server is up")
# Wait for server to be up
eventually(lambda: assert_equal(requests.get(f"{APP_URL}/state", timeout=5).status_code, 200, "Server did not respond with 200 OK within timeout"))
logger.info("Server is up")
# Wait for the pipeline to stabilize with dry=True:
# DockerContainerDeploy should return status="dry-run"
with brencher_page(APP_URL, env.id) as page:
eventually(
lambda: check_state(APP_URL, lambda s: assert_equal(next(
p for p in s[env.id]["pipeline"] if p["name"] == "SharedStateHolderInMemory"
)["status"]['dry'], True, "Pipeline step status is not 'running'")),
)
# Wait for the pipeline to stabilize with dry=True:
# DockerContainerDeploy should return status="dry-run"
with brencher_page(APP_URL, env.id) as page:
eventually(
lambda: check_state(APP_URL, lambda s: assert_equal(next(
p for p in s[env.id]["pipeline"] if p["name"] == "SharedStateHolderInMemory"
)["status"]['dry'], True, "Pipeline step status is not 'running'")),
)
# page.verify_pipeline_step_status("DockerContainerDeploy", "dry-run")
logger.info("Pipeline is in dry-run mode")
# page.verify_pipeline_step_status("DockerContainerDeploy", "dry-run")
logger.info("Pipeline is in dry-run mode")
# Verify no real container was created
docker_client = docker.from_env()
real_containers = docker_client.containers.list(filters={"name": CONTAINER_NAME}, all=True)
assert len(real_containers) == 0, "Container should NOT exist during dry run"
# Verify no real container was created
docker_client = docker.from_env()
real_containers = docker_client.containers.list(filters={"name": CONTAINER_NAME}, all=True)
assert len(real_containers) == 0, "Container should NOT exist during dry run"
# Disable dry run from the UI.
page.verify_dry_run_on()
page.set_dry_run_off()
page.verify_dry_run_off()
page.click_refresh() # TODO Remove it
# Disable dry run from the UI.
page.verify_dry_run_on()
page.set_dry_run_off()
page.verify_dry_run_off()
page.click_refresh() # TODO Remove it
# Wait for pipeline to re-run and actually deploy the container
eventually(
lambda: check_state(APP_URL, lambda s: assert_equal(next(
p for p in s[env.id]["pipeline"] if p["name"] =="SharedStateHolderInMemory"
)["status"]['dry'], False, "Pipeline step status is not 'running'")),
)
# Wait for pipeline to re-run and actually deploy the container
eventually(
lambda: check_state(APP_URL, lambda s: assert_equal({p["name"] for p in s[env.id]["pipeline"] if bool(p['is_running'])}, set(), "Pipeline still running")),
timeout=20,
interval=1.0,
)
# page.verify_pipeline_step_status("DockerContainerDeploy", "running")
logger.info("Container deployed and running after dry run disabled")
# Wait for pipeline to re-run and actually deploy the container
eventually(
lambda: check_state(APP_URL, lambda s: assert_equal(next(
p for p in s[env.id]["pipeline"] if p["name"] =="SharedStateHolderInMemory"
)["status"]['dry'], False, "Pipeline step status is not 'running'")),
)
# Wait for pipeline to re-run and actually deploy the container
eventually(
lambda: check_state(APP_URL, lambda s: assert_equal({p["name"] for p in s[env.id]["pipeline"] if bool(p['is_running'])}, set(), "Pipeline still running")),
timeout=20,
interval=1.0,
)
# page.verify_pipeline_step_status("DockerContainerDeploy", "running")
logger.info("Container deployed and running after dry run disabled")
# Final sanity: real container must exist and be running
containers = docker_client.containers.list(filters={"name": CONTAINER_NAME})
assert len(containers) == 1, "Container should exist and be running after deploy"
assert containers[0].status == "running"
# Final sanity: real container must exist and be running
containers = docker_client.containers.list(filters={"name": CONTAINER_NAME})
assert len(containers) == 1, "Container should exist and be running after deploy"
assert containers[0].status == "running"
finally:
app.stop()
server_thread.join(timeout=10)