From efe49119a314f8c32194a2993ece016c784af524 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 19 May 2026 19:02:30 +0000 Subject: [PATCH 1/7] Initial plan -- 2.54.0 From 6f3dd6ebd8a80164ba54bd2f7c5a66ee9104b664 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 19 May 2026 19:34:46 +0000 Subject: [PATCH 2/7] refactor app concurrency to asyncio taskgroup Agent-Logs-Url: https://github.com/rudolf1/brencher/sessions/9e719b58-ad2a-4eca-99bf-43c750f71a72 Co-authored-by: rudolf1 <5525769+rudolf1@users.noreply.github.com> --- backend/app.py | 90 ++++++++++++++++++++++++-------- backend/web.py | 50 +++++++++++++----- tests/test_app_asyncio.py | 50 ++++++++++++++++++ tests/test_containertest.py | 41 +++++++-------- tests/test_dry_run_playwright.py | 85 +++++++++++++++--------------- 5 files changed, 219 insertions(+), 97 deletions(-) create mode 100644 tests/test_app_asyncio.py diff --git a/backend/app.py b/backend/app.py index 0ced9f9..63b567e 100644 --- a/backend/app.py +++ b/backend/app.py @@ -1,8 +1,8 @@ +import asyncio import logging import os import signal import sys -import threading import traceback from typing import Any, Callable, Dict, List, Optional @@ -27,9 +27,12 @@ class App: def __init__(self, environments: Dict[str, Environment]) -> None: self.environments: Dict[str, Environment] = {id: wrap_in_cached(e) for id, e in environments.items()} - self.state_lock = threading.Lock() - self.environment_update_event = threading.Event() + self.environment_update_event = asyncio.Event() + self.environment_update_version = 0 self.emit_callback: Optional[Callable[[], None]] = None + self.reset_requested = False + self.shutdown_event = asyncio.Event() + self.web_app: Any = None def get_local_envs_to_emit(self) -> Dict[str, Dict[str, Any]]: env_dtos: Dict[str, Dict[str, Any]] = {} @@ -88,35 +91,80 @@ class App: return branches - def processing_thread(self) -> None: - while True: + async def processing_loop(self) -> None: + while not self.shutdown_event.is_set(): import processing emit = self.emit_callback or (lambda: None) + update_version = self.environment_update_version - with self.state_lock: - logger.info(f"Processing") - if processing.process_all_jobs(list(self.environments.values()), emit): - self.environment_update_event.wait(timeout=1 * 5) - else: - self.environment_update_event.wait(timeout=1 * 60) - self.environment_update_event.clear() + logger.info("Processing") + if self.reset_requested: + await asyncio.to_thread(processing.reset_caches, list(self.environments.values())) + self.reset_requested = False + has_error = await asyncio.to_thread(processing.process_all_jobs, list(self.environments.values()), emit) + if self.shutdown_event.is_set(): + return + if self.environment_update_version != update_version: + continue + timeout = 5 if has_error else 60 + try: + await asyncio.wait_for(self.environment_update_event.wait(), timeout=timeout) + except TimeoutError: + pass + self.environment_update_event.clear() - def run(self) -> None: - processing = threading.Thread(target=self.processing_thread) - processing.daemon = True - processing.start() + def notify_environment_update(self) -> None: + self.environment_update_version += 1 + self.environment_update_event.set() + + def request_reset(self) -> None: + self.reset_requested = True + + def stop(self) -> None: + self.shutdown_event.set() + self.notify_environment_update() + if self.web_app is not None: + self.web_app.stop() + + async def runAsync(self) -> None: + await self.processing_loop() + + async def runHeadlessAsync(self) -> None: + await self.processing_loop() def runHeadless(self) -> None: - """Run the processing loop on the current thread; blocks forever.""" - self.processing_thread() + """Run the processing loop until cancelled; blocks forever.""" + asyncio.run(self.runHeadlessAsync()) - def runWeb(self, port: int) -> None: + async def _run_web_server(self, web_app: Any) -> None: + await web_app.start_async() + raise _WebServerStopped() + + async def runWebAsync(self, port: int) -> None: import web web_app = web.WebApp(core=self, port=port) + self.web_app = web_app self.emit_callback = web_app.emit_envs - self.run() - web_app.start() + try: + async with asyncio.TaskGroup() as task_group: + task_group.create_task(self.processing_loop()) + task_group.create_task(self._run_web_server(web_app)) + except* _WebServerStopped: + pass + finally: + self.emit_callback = None + self.web_app = None + + def runWeb(self, port: int) -> None: + asyncio.run(self.runWebAsync(port)) + + def run(self) -> None: + asyncio.run(self.runAsync()) + + +class _WebServerStopped(Exception): + pass if __name__ == '__main__': cli_env_ids_list = sys.argv[1:] diff --git a/backend/web.py b/backend/web.py index 461ffe6..6f29501 100644 --- a/backend/web.py +++ b/backend/web.py @@ -11,11 +11,8 @@ from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import FileResponse from app import App -from enironment import get_step from utils import custom_json_dumps -from processing import reset_caches from secondary import SecondaryManager -from steps.step import CachingStep logger = logging.getLogger(__name__) @@ -48,11 +45,12 @@ class WebApp: self.ws_connections: Dict[WebSocket, Dict[str, Any]] = {} self._event_loop: Optional[asyncio.AbstractEventLoop] = None + self._server: Any = None self.secondaryManager: Optional[SecondaryManager] = None @asynccontextmanager async def lifespan(fastapi_app: FastAPI) -> AsyncIterator[None]: - self._event_loop = asyncio.get_event_loop() + self._event_loop = asyncio.get_running_loop() self.secondaryManager = SecondaryManager( urls=os.getenv("SECONDARY_BRENCHER", ""), on_update=lambda: self.broadcast_all(), @@ -94,10 +92,19 @@ class WebApp: merged = _merge_dicts(merged, connector.branches) return merged - # --- Async scheduling from sync threads --- + # --- Async scheduling from sync callbacks --- def _schedule_async(self, coro: Any) -> None: - if self._event_loop is not None and self._event_loop.is_running(): + if self._event_loop is None or not self._event_loop.is_running(): + coro.close() + return + try: + running_loop = asyncio.get_running_loop() + except RuntimeError: + running_loop = None + if running_loop is self._event_loop: + self._event_loop.create_task(coro) + else: asyncio.run_coroutine_threadsafe(coro, self._event_loop) # --- Broadcasting --- @@ -134,7 +141,7 @@ class WebApp: await self.broadcast("error", data) def emit_envs(self) -> None: - """Sync callback used by App's processing thread to push updates to clients.""" + """Sync callback used by App's processing loop to push updates to clients.""" self._schedule_async(self.broadcast_branches(self.get_global_branches_to_emit())) self._schedule_async(self.broadcast_environments(self.get_global_envs_to_emit())) @@ -179,7 +186,7 @@ class WebApp: await self.secondaryManager.send({"update": update_data}) id = update_data.get('id', '') if id == '': - reset_caches(list(self.core.environments.values())) + self.core.request_reset() elif id not in self.core.environments.keys() and id not in {j for it in self.secondaryManager or [] for j in it.environments.keys()}: logger.warning(f"Received update for unknown environment id {id}") continue @@ -196,11 +203,9 @@ class WebApp: env.state.set_dry(bool(update_data['dry']), expected_token) if env: - p = get_step(env.pipeline, type(env.state)) - if isinstance(p, CachingStep): - p.reset() + self.core.request_reset() logger.info(f"Updated environment {env.id} branches to {update_data.get('branches')}, dry={update_data.get('dry')}") - self.core.environment_update_event.set() + self.core.notify_environment_update() await self.broadcast_environments(self.get_global_envs_to_emit()) @@ -212,6 +217,23 @@ class WebApp: # --- Run --- - def start(self) -> None: + async def start_async(self) -> None: import uvicorn - uvicorn.run(self.app, host='0.0.0.0', port=self.port, log_level='info') + config = uvicorn.Config(self.app, host='0.0.0.0', port=self.port, log_level='info') + self._server = uvicorn.Server(config) + try: + await self._server.serve() + except asyncio.CancelledError: + self._server.should_exit = True + raise + + def start(self) -> None: + asyncio.run(self.start_async()) + + def stop(self) -> None: + if self._server is None: + return + if self._event_loop is not None and self._event_loop.is_running(): + self._event_loop.call_soon_threadsafe(setattr, self._server, "should_exit", True) + else: + self._server.should_exit = True diff --git a/tests/test_app_asyncio.py b/tests/test_app_asyncio.py new file mode 100644 index 0000000..92ff143 --- /dev/null +++ b/tests/test_app_asyncio.py @@ -0,0 +1,50 @@ +import asyncio +import sys +import types + +import pytest + +import processing +from app import App + + +class TestAppAsyncio: + @pytest.mark.asyncio + async def test_run_web_async_stops_processing_when_server_exits(self, monkeypatch: pytest.MonkeyPatch) -> None: + process_calls = 0 + emit_calls = 0 + processing_started = asyncio.Event() + loop = asyncio.get_running_loop() + web_app_args: dict[str, object] = {} + + def fake_process_all_jobs(environments: object, onupdate: object) -> bool: + nonlocal process_calls + process_calls += 1 + assert callable(onupdate) + onupdate() + loop.call_soon_threadsafe(processing_started.set) + return False + + class FakeWebApp: + def __init__(self, core: App, port: int) -> None: + web_app_args["core"] = core + web_app_args["port"] = port + + def emit_envs(self) -> None: + nonlocal emit_calls + emit_calls += 1 + + async def start_async(self) -> None: + await asyncio.wait_for(processing_started.wait(), timeout=1) + + monkeypatch.setattr(processing, "process_all_jobs", fake_process_all_jobs) + monkeypatch.setitem(sys.modules, "web", types.SimpleNamespace(WebApp=FakeWebApp)) + + app = App({}) + + await asyncio.wait_for(app.runWebAsync(5001), timeout=1) + + assert process_calls >= 1 + assert emit_calls >= 1 + assert web_app_args == {"core": app, "port": 5001} + assert app.emit_callback is None diff --git a/tests/test_containertest.py b/tests/test_containertest.py index 665627f..decdb57 100644 --- a/tests/test_containertest.py +++ b/tests/test_containertest.py @@ -1,7 +1,7 @@ import asyncio import json import logging -import threading +from contextlib import suppress from typing import Any, Generator import docker @@ -36,30 +36,29 @@ class TestDockerContainer: logger.info(f"Starting") app = App({"brencher_local1": brencher_local1.brencher_local1}) - # app.runWeb(5001) - processing = threading.Thread(target=lambda: app.runWeb(5001), daemon=True) - processing.start() + processing = asyncio.create_task(app.runWebAsync(5001)) + try: + eventually( + lambda: assert_equal(requests.get("http://localhost:5001/state", timeout=5000).status_code, 200, "Server did not respond with 200 OK within timeout"), + 20.0, + 1.0, + ) - # await asyncio.sleep(5000) + state_data = requests.get("http://localhost:5001/state", timeout=5000).json() + logger.info(f"Application state: {json.dumps(state_data, indent=2)}") - eventually( - lambda: assert_equal(requests.get("http://localhost:5001/state", timeout=5000).status_code, 200, "Server did not respond with 200 OK within timeout"), - 20.0, - 1.0, - ) + branches_data = requests.get("http://localhost:5001/branches", timeout=5000).json() + logger.info(f"Application branches: {json.dumps(branches_data, indent=2)}") + assert "brencher_local1" in state_data, "Missing 'brencher_local1' field in response" + assert len(state_data["brencher_local1"]["pipeline"]) > 0, "Steps defined for 'brencher_local1' are empty" - state_data = requests.get("http://localhost:5001/state", timeout=5000).json() - logger.info(f"Application state: {json.dumps(state_data, indent=2)}") - - branches_data = requests.get("http://localhost:5001/branches", timeout=5000).json() - logger.info(f"Application branches: {json.dumps(branches_data, indent=2)}") - - assert "brencher_local1" in state_data, "Missing 'brencher_local1' field in response" - assert len(state_data["brencher_local1"]["pipeline"]) > 0, "Steps defined for 'brencher_local1' are empty" - - assert "brencher_local1" in branches_data, "Missing 'brencher_local1' field in response" - assert len(branches_data["brencher_local1"]) > 0, "No branches found for 'brencher_local1'" + assert "brencher_local1" in branches_data, "Missing 'brencher_local1' field in response" + assert len(branches_data["brencher_local1"]) > 0, "No branches found for 'brencher_local1'" + finally: + processing.cancel() + with suppress(asyncio.CancelledError): + await processing # TODO Verify DockerContainerDeploy step status is running and has correct image and ports # { diff --git a/tests/test_dry_run_playwright.py b/tests/test_dry_run_playwright.py index e4d0894..44ccc72 100644 --- a/tests/test_dry_run_playwright.py +++ b/tests/test_dry_run_playwright.py @@ -43,50 +43,53 @@ class TestDryRunPlaywright: app = App({env.id: env}) server_thread = threading.Thread(target=lambda: app.runWeb(APP_PORT), daemon=True) server_thread.start() + try: + # Wait for server to be up + eventually(lambda: assert_equal(requests.get(f"{APP_URL}/state", timeout=5).status_code, 200, "Server did not respond with 200 OK within timeout")) + logger.info("Server is up") - # Wait for server to be up - eventually(lambda: assert_equal(requests.get(f"{APP_URL}/state", timeout=5).status_code, 200, "Server did not respond with 200 OK within timeout")) - logger.info("Server is up") + # Wait for the pipeline to stabilize with dry=True: + # DockerContainerDeploy should return status="dry-run" + with brencher_page(APP_URL, env.id) as page: + eventually( + lambda: check_state(APP_URL, lambda s: assert_equal(next( + p for p in s[env.id]["pipeline"] if p["name"] == "SharedStateHolderInMemory" + )["status"]['dry'], True, "Pipeline step status is not 'running'")), + ) - # Wait for the pipeline to stabilize with dry=True: - # DockerContainerDeploy should return status="dry-run" - with brencher_page(APP_URL, env.id) as page: - eventually( - lambda: check_state(APP_URL, lambda s: assert_equal(next( - p for p in s[env.id]["pipeline"] if p["name"] == "SharedStateHolderInMemory" - )["status"]['dry'], True, "Pipeline step status is not 'running'")), - ) + # page.verify_pipeline_step_status("DockerContainerDeploy", "dry-run") + logger.info("Pipeline is in dry-run mode") - # page.verify_pipeline_step_status("DockerContainerDeploy", "dry-run") - logger.info("Pipeline is in dry-run mode") + # Verify no real container was created + docker_client = docker.from_env() + real_containers = docker_client.containers.list(filters={"name": CONTAINER_NAME}, all=True) + assert len(real_containers) == 0, "Container should NOT exist during dry run" - # Verify no real container was created - docker_client = docker.from_env() - real_containers = docker_client.containers.list(filters={"name": CONTAINER_NAME}, all=True) - assert len(real_containers) == 0, "Container should NOT exist during dry run" + # Disable dry run from the UI. + page.verify_dry_run_on() + page.set_dry_run_off() + page.verify_dry_run_off() + page.click_refresh() # TODO Remove it - # Disable dry run from the UI. - page.verify_dry_run_on() - page.set_dry_run_off() - page.verify_dry_run_off() - page.click_refresh() # TODO Remove it + # Wait for pipeline to re-run and actually deploy the container + eventually( + lambda: check_state(APP_URL, lambda s: assert_equal(next( + p for p in s[env.id]["pipeline"] if p["name"] =="SharedStateHolderInMemory" + )["status"]['dry'], False, "Pipeline step status is not 'running'")), + ) + # Wait for pipeline to re-run and actually deploy the container + eventually( + lambda: check_state(APP_URL, lambda s: assert_equal({p["name"] for p in s[env.id]["pipeline"] if bool(p['is_running'])}, set(), "Pipeline still running")), + timeout=20, + interval=1.0, + ) + # page.verify_pipeline_step_status("DockerContainerDeploy", "running") + logger.info("Container deployed and running after dry run disabled") - # Wait for pipeline to re-run and actually deploy the container - eventually( - lambda: check_state(APP_URL, lambda s: assert_equal(next( - p for p in s[env.id]["pipeline"] if p["name"] =="SharedStateHolderInMemory" - )["status"]['dry'], False, "Pipeline step status is not 'running'")), - ) - # Wait for pipeline to re-run and actually deploy the container - eventually( - lambda: check_state(APP_URL, lambda s: assert_equal({p["name"] for p in s[env.id]["pipeline"] if bool(p['is_running'])}, set(), "Pipeline still running")), - timeout=20, - interval=1.0, - ) - # page.verify_pipeline_step_status("DockerContainerDeploy", "running") - logger.info("Container deployed and running after dry run disabled") - - # Final sanity: real container must exist and be running - containers = docker_client.containers.list(filters={"name": CONTAINER_NAME}) - assert len(containers) == 1, "Container should exist and be running after deploy" - assert containers[0].status == "running" + # Final sanity: real container must exist and be running + containers = docker_client.containers.list(filters={"name": CONTAINER_NAME}) + assert len(containers) == 1, "Container should exist and be running after deploy" + assert containers[0].status == "running" + finally: + app.stop() + server_thread.join(timeout=10) -- 2.54.0 From 1acea859348399047a8f116a1ecdeae40362edf3 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 19 May 2026 19:35:59 +0000 Subject: [PATCH 3/7] add async shutdown cleanup and review fixes Agent-Logs-Url: https://github.com/rudolf1/brencher/sessions/9e719b58-ad2a-4eca-99bf-43c750f71a72 Co-authored-by: rudolf1 <5525769+rudolf1@users.noreply.github.com> --- backend/app.py | 2 ++ backend/web.py | 1 + tests/test_app_asyncio.py | 6 ++++-- tests/test_containertest.py | 1 + 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/backend/app.py b/backend/app.py index 63b567e..ee45d08 100644 --- a/backend/app.py +++ b/backend/app.py @@ -105,6 +105,7 @@ class App: has_error = await asyncio.to_thread(processing.process_all_jobs, list(self.environments.values()), emit) if self.shutdown_event.is_set(): return + # Skip the wait when a newer update arrived while the current processing pass was running. if self.environment_update_version != update_version: continue timeout = 5 if has_error else 60 @@ -138,6 +139,7 @@ class App: asyncio.run(self.runHeadlessAsync()) async def _run_web_server(self, web_app: Any) -> None: + """Raise a sentinel exception when the server stops so TaskGroup cancels processing.""" await web_app.start_async() raise _WebServerStopped() diff --git a/backend/web.py b/backend/web.py index 6f29501..74c961c 100644 --- a/backend/web.py +++ b/backend/web.py @@ -96,6 +96,7 @@ class WebApp: def _schedule_async(self, coro: Any) -> None: if self._event_loop is None or not self._event_loop.is_running(): + logger.warning("Dropping scheduled coroutine because the web event loop is not running") coro.close() return try: diff --git a/tests/test_app_asyncio.py b/tests/test_app_asyncio.py index 92ff143..1fe7e26 100644 --- a/tests/test_app_asyncio.py +++ b/tests/test_app_asyncio.py @@ -1,9 +1,11 @@ import asyncio import sys import types +from typing import Callable import pytest +from enironment import Environment import processing from app import App @@ -17,10 +19,10 @@ class TestAppAsyncio: loop = asyncio.get_running_loop() web_app_args: dict[str, object] = {} - def fake_process_all_jobs(environments: object, onupdate: object) -> bool: + def fake_process_all_jobs(environments: list[Environment], onupdate: Callable[[], None]) -> bool: nonlocal process_calls process_calls += 1 - assert callable(onupdate) + assert environments == [] onupdate() loop.call_soon_threadsafe(processing_started.set) return False diff --git a/tests/test_containertest.py b/tests/test_containertest.py index decdb57..5fe5279 100644 --- a/tests/test_containertest.py +++ b/tests/test_containertest.py @@ -59,6 +59,7 @@ class TestDockerContainer: processing.cancel() with suppress(asyncio.CancelledError): await processing + assert processing.cancelled() # TODO Verify DockerContainerDeploy step status is running and has correct image and ports # { -- 2.54.0 From b2782ee22070f6b0da9052b14599e2831326f532 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 19 May 2026 19:36:56 +0000 Subject: [PATCH 4/7] clarify async taskgroup behavior Agent-Logs-Url: https://github.com/rudolf1/brencher/sessions/9e719b58-ad2a-4eca-99bf-43c750f71a72 Co-authored-by: rudolf1 <5525769+rudolf1@users.noreply.github.com> --- backend/app.py | 3 ++- backend/web.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/app.py b/backend/app.py index ee45d08..076f55e 100644 --- a/backend/app.py +++ b/backend/app.py @@ -111,7 +111,7 @@ class App: timeout = 5 if has_error else 60 try: await asyncio.wait_for(self.environment_update_event.wait(), timeout=timeout) - except TimeoutError: + except asyncio.TimeoutError: pass self.environment_update_event.clear() @@ -149,6 +149,7 @@ class App: self.web_app = web_app self.emit_callback = web_app.emit_envs try: + # Keep processing and web serving coupled so shutdown or fatal failure tears down the whole app. async with asyncio.TaskGroup() as task_group: task_group.create_task(self.processing_loop()) task_group.create_task(self._run_web_server(web_app)) diff --git a/backend/web.py b/backend/web.py index 74c961c..1bb7b72 100644 --- a/backend/web.py +++ b/backend/web.py @@ -235,6 +235,6 @@ class WebApp: if self._server is None: return if self._event_loop is not None and self._event_loop.is_running(): - self._event_loop.call_soon_threadsafe(setattr, self._server, "should_exit", True) + self._event_loop.call_soon_threadsafe(lambda: setattr(self._server, "should_exit", True)) else: self._server.should_exit = True -- 2.54.0 From ab01fca8417326885fa95f21cbbe009ef7e239b0 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 19 May 2026 20:06:11 +0000 Subject: [PATCH 5/7] stabilize dry-run playwright integration Agent-Logs-Url: https://github.com/rudolf1/brencher/sessions/d019d010-3ca0-4a73-8826-620eef58edab Co-authored-by: rudolf1 <5525769+rudolf1@users.noreply.github.com> --- Dockerfile_test_app | 12 +++--------- tests/configs/nginx.py | 13 +++++++++++++ tests/test_dry_run_playwright.py | 20 +++++++++++++------- 3 files changed, 29 insertions(+), 16 deletions(-) diff --git a/Dockerfile_test_app b/Dockerfile_test_app index f1bf22f..a4bfb6b 100644 --- a/Dockerfile_test_app +++ b/Dockerfile_test_app @@ -1,14 +1,10 @@ FROM nginx:1.27-alpine -COPY .git /tmp/repo/.git - # Configure nginx to listen on 5001 to match container port expectations. RUN sed -i 's/listen 80;/listen 5001;/' /etc/nginx/conf.d/default.conf -# Build a custom landing page directly in the image and inject git revision. +# Build a custom landing page directly in the image. RUN set -eux; \ - apk add --no-cache --virtual .git-deps git; \ - GIT_REVISION="$(git --git-dir=/tmp/repo/.git rev-parse --short HEAD 2>/dev/null || echo unknown)"; \ { \ echo ''; \ echo ''; \ @@ -21,11 +17,9 @@ RUN set -eux; \ echo '
Custom landing page for the Brencher test app container.
'; \ echo 'Port: 5001
'; \ - echo "Git revision: ${GIT_REVISION}
"; \ + echo 'Git revision: unknown
'; \ echo ''; \ echo '