diff --git a/backend/app.py b/backend/app.py index 542f4f0..8b85aca 100644 --- a/backend/app.py +++ b/backend/app.py @@ -7,7 +7,7 @@ import sys import threading import traceback from dataclasses import asdict, replace -from typing import Any, Dict, List, Optional, Set, Tuple, TypeVar +from typing import Any, Dict, List, Optional, Set, TypeVar import websockets from dotenv import load_dotenv @@ -64,7 +64,7 @@ environments_slaves: Dict[str, Any] = {} branches_slaves: Dict[str, Dict[str, Any]] = {} state_lock = threading.Lock() -# Single set of WebSocket connections +# Active WebSocket connections ws_connections: Set[WebSocket] = set() # Event loop reference for thread-safe async calls @@ -206,13 +206,16 @@ def _schedule_async(coro: Any) -> None: # --- WebSocket Broadcasting Functions --- async def broadcast(event: str, data: Any) -> None: - """Broadcast a message to all connected WebSocket clients.""" + """Broadcast a message to all connected WebSocket clients that have not seen this data yet.""" disconnected = set() message = custom_json_dumps({event: data}) for websocket in list(ws_connections): + if websocket.state.last_payloads.get(event) == message: + continue try: await websocket.send_text(message) + websocket.state.last_payloads[event] = message except Exception as e: logger.error(f"Error sending to websocket: {e}") disconnected.add(websocket) @@ -238,16 +241,17 @@ async def broadcast_error(data: Any) -> None: @app.websocket("/ws") async def websocket_endpoint(websocket: WebSocket) -> None: await websocket.accept() + websocket.state.last_payloads: Dict[str, str] = {} ws_connections.add(websocket) try: - # Send initial state on connect - await websocket.send_text(custom_json_dumps({ - "branches": get_global_branches_to_emit(), - })) - await websocket.send_text(custom_json_dumps({ - "environments": get_global_envs_to_emit(), - })) + # Send initial state on connect and record it so duplicate broadcasts are suppressed + branches_payload = custom_json_dumps({"branches": get_global_branches_to_emit()}) + envs_payload = custom_json_dumps({"environments": get_global_envs_to_emit()}) + await websocket.send_text(branches_payload) + websocket.state.last_payloads["branches"] = branches_payload + await websocket.send_text(envs_payload) + websocket.state.last_payloads["environments"] = envs_payload while True: data = await websocket.receive_text() diff --git a/backend/steps/step.py b/backend/steps/step.py index 2b3d032..87d1adf 100644 --- a/backend/steps/step.py +++ b/backend/steps/step.py @@ -1,4 +1,6 @@ -from typing import TypeVar, Generic +import hashlib +import json +from typing import TypeVar, Generic, Any from enironment import AbstractStep @@ -10,26 +12,47 @@ class NotReadyException(BaseException): super().__init__(message) +def _stable_hash(obj: Any) -> str: + """Compute a stable SHA-256 hash of an object via JSON serialization.""" + try: + serialized = json.dumps(obj, sort_keys=True, default=repr) + except Exception: + serialized = repr(obj) + return hashlib.sha256(serialized.encode()).hexdigest() + + class CachingStep(AbstractStep[T], Generic[T]): _result: T | BaseException - reset_cache: bool = False + _input_hash: str | None def __init__(self, step: AbstractStep[T]) -> None: super().__init__(n=step.name) self.step = step self._result = NotReadyException(f"No result yet for {self.step.name}") + self._input_hash = None + + def _compute_input_hash(self) -> str: + """Compute a stable hash of all AbstractStep dependency outputs.""" + inputs: dict[str, Any] = {} + for attr_name, attr_value in vars(self.step).items(): + if isinstance(attr_value, AbstractStep): + try: + inputs[attr_name] = attr_value.progress() + except BaseException as e: + inputs[attr_name] = repr(e) + return _stable_hash(inputs) def progress(self) -> T: - if self.reset_cache or self._result is None or isinstance(self._result, BaseException): + current_hash = self._compute_input_hash() + if self._input_hash != current_hash or isinstance(self._result, BaseException): try: self._result = self.step.progress() except BaseException as e: self._result = e - self.reset_cache = False + self._input_hash = current_hash if isinstance(self._result, BaseException): raise self._result from None - else: - return self._result + return self._result def reset(self) -> None: - self.reset_cache = True + self._input_hash = None diff --git a/blank.txt b/blank.txt new file mode 100644 index 0000000..e69de29 diff --git a/frontend/app.js b/frontend/app.js index 965b3c3..354a1dc 100644 --- a/frontend/app.js +++ b/frontend/app.js @@ -267,7 +267,10 @@ function renderJobs() { jobsList.innerHTML = '
No jobs found.
'; return; } - jobsList.innerHTML = environmentsRaw.map((envObj) => { + jobsList.innerHTML = environmentsRaw + .slice() + .sort((a, b) => (a.name || a.id || '').localeCompare(b.name || b.id || '')) + .map((envObj) => { var linksHtml = "" const jobsArr = envObj.pipeline || []; if (Array.isArray(jobsArr)) { diff --git a/tests/test_caching_step.py b/tests/test_caching_step.py new file mode 100644 index 0000000..346b363 --- /dev/null +++ b/tests/test_caching_step.py @@ -0,0 +1,167 @@ +""" +Unit tests for CachingStep input-hash-based cache invalidation. +""" +from typing import Any + +import pytest + +from enironment import AbstractStep, Environment +from steps.step import CachingStep, NotReadyException + + +class ConstantStep(AbstractStep[str]): + """Step that always returns a fixed value, counting calls.""" + + def __init__(self, value: str) -> None: + super().__init__() + self.value = value + self.call_count = 0 + + def progress(self) -> str: + self.call_count += 1 + return self.value + + +class DependentStep(AbstractStep[str]): + """Step that depends on another AbstractStep and appends a suffix.""" + + def __init__(self, upstream: AbstractStep[str], suffix: str = "-processed") -> None: + super().__init__() + self.upstream = upstream + self.suffix = suffix + self.call_count = 0 + + def progress(self) -> str: + self.call_count += 1 + return self.upstream.progress() + self.suffix + + +class FailingStep(AbstractStep[str]): + """Step that always raises an exception.""" + + def __init__(self) -> None: + super().__init__() + self.call_count = 0 + + def progress(self) -> str: + self.call_count += 1 + raise RuntimeError("step failed") + + +def _make_env(steps: list[AbstractStep[Any]]) -> Environment: + return Environment(id="test", branches=[], dry=True, repo="repo", pipeline=steps) + + +class TestCachingStep: + def test_executes_on_first_call(self) -> None: + inner = ConstantStep("hello") + cached = CachingStep(inner) + _make_env([cached]) + + result = cached.progress() + + assert result == "hello" + assert inner.call_count == 1 + + def test_caches_result_when_inputs_unchanged(self) -> None: + inner = ConstantStep("hello") + cached = CachingStep(inner) + _make_env([cached]) + + cached.progress() + cached.progress() + cached.progress() + + assert inner.call_count == 1 + + def test_reexecutes_when_upstream_output_changes(self) -> None: + upstream = ConstantStep("v1") + inner = DependentStep(upstream) + cached_upstream = CachingStep(upstream) + cached = CachingStep(inner) + _make_env([cached_upstream, cached]) + + # Wire up: replace inner's `upstream` reference with the cached wrapper + inner.upstream = cached_upstream + + result1 = cached.progress() + assert result1 == "v1-processed" + assert inner.call_count == 1 + + # Change the upstream value so the hash changes + upstream.value = "v2" + # Force the upstream cache to re-execute so it returns the new value + cached_upstream.reset() + + result2 = cached.progress() + assert result2 == "v2-processed" + assert inner.call_count == 2 + + def test_does_not_reexecute_when_upstream_output_unchanged(self) -> None: + upstream = ConstantStep("stable") + inner = DependentStep(upstream) + cached_upstream = CachingStep(upstream) + cached = CachingStep(inner) + _make_env([cached_upstream, cached]) + + inner.upstream = cached_upstream + + cached.progress() + cached.progress() + cached.progress() + + assert inner.call_count == 1 + + def test_reset_forces_reexecution(self) -> None: + inner = ConstantStep("hello") + cached = CachingStep(inner) + _make_env([cached]) + + cached.progress() + assert inner.call_count == 1 + + cached.reset() + cached.progress() + assert inner.call_count == 2 + + def test_retries_after_exception(self) -> None: + inner = FailingStep() + cached = CachingStep(inner) + _make_env([cached]) + + with pytest.raises(RuntimeError): + cached.progress() + assert inner.call_count == 1 + + # Second call should retry because the result is still an exception + with pytest.raises(RuntimeError): + cached.progress() + assert inner.call_count == 2 + + def test_initial_state_raises_not_ready(self) -> None: + inner = ConstantStep("hello") + cached = CachingStep(inner) + # Do NOT set up env – but CachingStep itself should start with NotReadyException + assert isinstance(cached._result, NotReadyException) + + def test_leaf_step_executes_once(self) -> None: + """A step with no AbstractStep dependencies uses a constant hash; executes only once.""" + inner = ConstantStep("leaf") + cached = CachingStep(inner) + _make_env([cached]) + + cached.progress() + cached.progress() + + assert inner.call_count == 1 + + def test_reset_clears_hash(self) -> None: + inner = ConstantStep("x") + cached = CachingStep(inner) + _make_env([cached]) + + cached.progress() + assert cached._input_hash is not None + + cached.reset() + assert cached._input_hash is None