Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
68af144f7a | ||
|
|
45898cc4e2 | ||
|
|
5d9cbe8df6 | ||
|
|
2a14d70fab | ||
|
|
234ffdd634 | ||
|
|
c3e0bff795 | ||
|
|
40544a3e42 | ||
|
|
e44c78e569 |
@@ -0,0 +1,53 @@
|
||||
---
|
||||
name: playwright-brencher
|
||||
description: "Write or refactor Playwright tests for brencher UI. Use when creating page helpers, extracting browser code from tests, or enforcing set_/verify_ method naming."
|
||||
---
|
||||
|
||||
# Playwright Brencher Conventions
|
||||
|
||||
Use this skill when writing or refactoring Playwright tests in this repository.
|
||||
|
||||
## Required API Shape
|
||||
|
||||
Create and use a context manager in this style:
|
||||
|
||||
```python
|
||||
with brencher_page(APP_URL) as page:
|
||||
page.verify_dry_run_on()
|
||||
page.set_dry_run_off()
|
||||
page.verify_dry_run_off()
|
||||
```
|
||||
|
||||
## Naming Rules
|
||||
|
||||
1. Action methods that change UI/application state must use `set_...` names.
|
||||
2. Verification methods (assertions, waits, checks) must use `verify_...` names.
|
||||
3. Keep low-level selector helpers private (prefix `_`).
|
||||
4. Prefer one page-object class per screen/area (`BrencherPage` for the current app page).
|
||||
|
||||
## Structure Rules
|
||||
|
||||
1. Put shared Playwright helpers in `tests/playwright_helper.py`.
|
||||
2. Keep tests focused on scenario flow; move DOM selectors and waiting logic into page helper methods.
|
||||
3. Use context manager functions to own Playwright/browser lifecycle.
|
||||
4. Inline trivial one-use logic where it improves readability and does not duplicate behavior.
|
||||
|
||||
## Method Guidance
|
||||
|
||||
Recommended style for page object methods:
|
||||
|
||||
- `set_*`: click/toggle/type actions.
|
||||
- `verify_*`: assert visible state, button title/text, or wait until target state is reached.
|
||||
|
||||
Example method names:
|
||||
|
||||
- `set_dry_run_off`
|
||||
- `verify_dry_run_on`
|
||||
- `verify_dry_run_off`
|
||||
|
||||
## Review Checklist
|
||||
|
||||
1. Test uses `with brencher_page(...) as page` style.
|
||||
2. No direct `sync_playwright()` block remains in the test body when helper abstraction is intended.
|
||||
3. Action and verification methods follow `set_` / `verify_` naming.
|
||||
4. Selector details are encapsulated in private helper methods.
|
||||
@@ -12,13 +12,12 @@ on:
|
||||
|
||||
jobs:
|
||||
build:
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
python-version: [
|
||||
# "3.11",
|
||||
# "3.11.3",
|
||||
"3.12.3",
|
||||
"3.13",
|
||||
"3.14",
|
||||
@@ -31,7 +30,6 @@ jobs:
|
||||
"flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics",
|
||||
# "ty check backend",
|
||||
"mypy --no-incremental --config-file mypy.ini backend tests",
|
||||
"pytest --ignore=tests/integration",
|
||||
]
|
||||
|
||||
steps:
|
||||
@@ -43,7 +41,42 @@ jobs:
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
python -m pip install flake8 pytest ty
|
||||
pip install -r requirements.txt
|
||||
pip install -r requirements-dev.txt
|
||||
- name: Run checker
|
||||
run: ${{ matrix.checker }}
|
||||
|
||||
tests:
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
python-version: [
|
||||
# "3.11.3",
|
||||
"3.12.3",
|
||||
"3.13",
|
||||
"3.14",
|
||||
# "3.15",
|
||||
]
|
||||
checker: [
|
||||
"pytest",
|
||||
]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Set up Python ${{ matrix.python-version }}, ${{ matrix.checker }}
|
||||
uses: actions/setup-python@v3
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
# sudo apt-get update
|
||||
# sudo apt-get upgrade -y
|
||||
# sudo apt-get install -y --no-install-recommends git docker.io
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r requirements-dev.txt
|
||||
playwright install --with-deps chromium
|
||||
- name: Run checker
|
||||
run: "pytest"
|
||||
|
||||
|
||||
|
||||
+28
-1
@@ -1,16 +1,40 @@
|
||||
# Use official Python image with uv pre-installed
|
||||
FROM python:3.12.3-alpine
|
||||
FROM python:3.12.3-slim AS typecheck
|
||||
|
||||
# Install uv (if not present)
|
||||
RUN pip install uv
|
||||
|
||||
# Set workdir
|
||||
WORKDIR /app
|
||||
|
||||
# Install development dependencies and run mypy before the runtime image is built.
|
||||
COPY requirements.txt requirements-dev.txt mypy.ini ./
|
||||
|
||||
RUN uv venv .venv && \
|
||||
uv pip install -r requirements-dev.txt
|
||||
|
||||
COPY backend/ ./backend/
|
||||
COPY tests/ ./tests/
|
||||
|
||||
RUN . .venv/bin/activate && \
|
||||
mypy --no-incremental --config-file mypy.ini backend tests && \
|
||||
touch /tmp/typecheck.done
|
||||
|
||||
# Use official Python image with uv pre-installed
|
||||
FROM python:3.12.3-alpine AS base
|
||||
|
||||
# Install uv (if not present)
|
||||
RUN pip install uv
|
||||
# Install git
|
||||
RUN apk update && apk upgrade
|
||||
RUN apk add --no-cache git docker-cli
|
||||
|
||||
# Set workdir
|
||||
WORKDIR /app
|
||||
|
||||
# Copy requirements and install dependencies with uv
|
||||
COPY requirements.txt ./
|
||||
|
||||
RUN uv venv .venv && \
|
||||
uv pip install -r requirements.txt
|
||||
|
||||
@@ -21,6 +45,9 @@ COPY frontend/ ./frontend/
|
||||
# Expose port for the web server
|
||||
EXPOSE 5001
|
||||
|
||||
# Ensure the mypy pre-build stage is executed.
|
||||
COPY --from=typecheck /tmp/typecheck.done /tmp/typecheck.done
|
||||
|
||||
# Set environment variables
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
ENV PATH="/app/.venv/bin:$PATH"
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
FROM nginx:1.27-alpine
|
||||
|
||||
COPY .git /tmp/repo/.git
|
||||
|
||||
# Configure nginx to listen on 5001 to match container port expectations.
|
||||
RUN sed -i 's/listen 80;/listen 5001;/' /etc/nginx/conf.d/default.conf
|
||||
|
||||
# Build a custom landing page directly in the image and inject git revision.
|
||||
RUN set -eux; \
|
||||
apk add --no-cache --virtual .git-deps git; \
|
||||
GIT_REVISION="$(git --git-dir=/tmp/repo/.git rev-parse --short HEAD 2>/dev/null || echo unknown)"; \
|
||||
{ \
|
||||
echo '<!doctype html>'; \
|
||||
echo '<html lang="en">'; \
|
||||
echo '<head>'; \
|
||||
echo ' <meta charset="utf-8">'; \
|
||||
echo ' <meta name="viewport" content="width=device-width, initial-scale=1">'; \
|
||||
echo ' <title>Brencher Test App</title>'; \
|
||||
echo '</head>'; \
|
||||
echo '<body>'; \
|
||||
echo ' <h1>Nginx Is Running</h1>'; \
|
||||
echo ' <p>Custom landing page for the Brencher test app container.</p>'; \
|
||||
echo ' <p>Port: 5001</p>'; \
|
||||
echo " <p>Git revision: ${GIT_REVISION}</p>"; \
|
||||
echo '</body>'; \
|
||||
echo '</html>'; \
|
||||
} > /usr/share/nginx/html/index.html; \
|
||||
rm -rf /tmp/repo/.git; \
|
||||
apk del .git-deps
|
||||
|
||||
EXPOSE 5001
|
||||
+84
-402
@@ -1,23 +1,16 @@
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import signal
|
||||
import sys
|
||||
import threading
|
||||
import traceback
|
||||
from dataclasses import asdict, replace
|
||||
from typing import Any, Dict, List, Optional, Set, TypeVar
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
import websockets
|
||||
from discover_envs import build_environments, parse_arguments
|
||||
from utils import sigchld_handler
|
||||
from dotenv import load_dotenv
|
||||
from fastapi import FastAPI, Response, WebSocket, WebSocketDisconnect
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import FileResponse
|
||||
|
||||
from configs.gmail_mcp import gmail_mcp
|
||||
from enironment import AbstractStep, Environment, wrap_in_cached
|
||||
from processing import reset_caches
|
||||
from enironment import Environment, wrap_in_cached, SharedStateHolder, get_step
|
||||
from steps.git import GitClone
|
||||
from steps.step import CachingStep
|
||||
|
||||
@@ -30,421 +23,100 @@ load_dotenv("local.env")
|
||||
load_dotenv('/run/secrets/brencher-secrets')
|
||||
|
||||
|
||||
class DataclassJSONEncoder(json.JSONEncoder):
|
||||
def default(self, o: Any) -> Any:
|
||||
if hasattr(o, '__dataclass_fields__'):
|
||||
return asdict(o)
|
||||
if isinstance(o, BaseException):
|
||||
return str(o)
|
||||
try:
|
||||
return super().default(o)
|
||||
except TypeError:
|
||||
return str(o)
|
||||
class App:
|
||||
|
||||
def __init__(self, environments: Dict[str, Environment]) -> None:
|
||||
self.environments: Dict[str, Environment] = {id: wrap_in_cached(e) for id, e in environments.items()}
|
||||
self.state_lock = threading.Lock()
|
||||
self.environment_update_event = threading.Event()
|
||||
self.emit_callback: Optional[Callable[[], None]] = None
|
||||
|
||||
def custom_json_dumps(obj: Any) -> str:
|
||||
return json.dumps(obj, cls=DataclassJSONEncoder)
|
||||
def get_local_envs_to_emit(self) -> Dict[str, Dict[str, Any]]:
|
||||
env_dtos: Dict[str, Dict[str, Any]] = {}
|
||||
for env in self.environments.values():
|
||||
pipeline_state: List[Dict[str, Any]] = []
|
||||
for r in env.pipeline:
|
||||
try:
|
||||
if isinstance(r, CachingStep):
|
||||
result = r._result
|
||||
else:
|
||||
result = r.progress()
|
||||
|
||||
|
||||
app = FastAPI()
|
||||
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=["*"],
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
FRONTEND_DIR = os.path.join(os.path.dirname(__file__), '../frontend')
|
||||
|
||||
# In-memory state
|
||||
environments: Dict[str, Environment] = {}
|
||||
environments_slaves: Dict[str, Any] = {}
|
||||
branches_slaves: Dict[str, Dict[str, Any]] = {}
|
||||
state_lock = threading.Lock()
|
||||
|
||||
# Active WebSocket connections
|
||||
ws_connections: Dict[WebSocket, Dict[str, Any]] = {}
|
||||
|
||||
# Event loop reference for thread-safe async calls
|
||||
_event_loop: Optional[asyncio.AbstractEventLoop] = None
|
||||
|
||||
# Slave connection state
|
||||
slave_ws_task: Optional[asyncio.Task[None]] = None
|
||||
slave_send_queue: asyncio.Queue[Dict[str, Any]] = asyncio.Queue()
|
||||
slave_url = os.getenv('SLAVE_BRENCHER')
|
||||
|
||||
environment_update_event = threading.Event()
|
||||
|
||||
|
||||
# --- Static File Serving ---
|
||||
|
||||
@app.get("/")
|
||||
async def serve_index() -> FileResponse:
|
||||
return FileResponse(os.path.join(FRONTEND_DIR, 'index.html'))
|
||||
|
||||
|
||||
@app.get("/state")
|
||||
async def serve_state() -> Response:
|
||||
return Response(content=custom_json_dumps(get_global_envs_to_emit()), media_type="application/json")
|
||||
|
||||
|
||||
@app.get("/branches")
|
||||
async def serve_branches_route() -> Response:
|
||||
return Response(content=custom_json_dumps(get_global_branches_to_emit()), media_type="application/json")
|
||||
|
||||
|
||||
@app.get("/{path:path}")
|
||||
async def serve_static(path: str) -> FileResponse:
|
||||
file_path = os.path.join(FRONTEND_DIR, path)
|
||||
if os.path.exists(file_path) and os.path.isfile(file_path):
|
||||
return FileResponse(file_path)
|
||||
return FileResponse(os.path.join(FRONTEND_DIR, 'index.html'))
|
||||
|
||||
|
||||
# --- Utility Functions ---
|
||||
|
||||
T = TypeVar('T')
|
||||
|
||||
|
||||
def merge_dicts(a: Dict[str, T], b: Dict[str, T]) -> Dict[str, T]:
|
||||
result: Dict[str, T] = {}
|
||||
for k in (a.keys() | b.keys()):
|
||||
if k not in a or k not in b:
|
||||
result[k] = a.get(k, b.get(k)) # type: ignore[assignment]
|
||||
continue
|
||||
if (
|
||||
isinstance(a[k], dict)
|
||||
and isinstance(b[k], dict)
|
||||
):
|
||||
result[k] = merge_dicts(a[k], b[k]) # type: ignore[arg-type,assignment]
|
||||
else:
|
||||
result[k] = b[k]
|
||||
return result
|
||||
|
||||
|
||||
def get_local_envs_to_emit() -> Dict[str, Dict[str, Any]]:
|
||||
env_dtos: Dict[str, Dict[str, Any]] = {}
|
||||
for env in environments.values():
|
||||
pipeline_state: List[Dict[str, Any]] = []
|
||||
for r in env.pipeline:
|
||||
try:
|
||||
if isinstance(r, CachingStep):
|
||||
result = r._result
|
||||
else:
|
||||
result = r.progress()
|
||||
|
||||
if isinstance(result, BaseException):
|
||||
stack = traceback.format_exception(type(result), result, result.__traceback__)
|
||||
if isinstance(result, BaseException):
|
||||
stack = traceback.format_exception(type(result), result, result.__traceback__)
|
||||
pipeline_state.append({
|
||||
"name": r.name,
|
||||
"status": [str(result), stack],
|
||||
"error": True,
|
||||
"is_running": True,
|
||||
})
|
||||
else:
|
||||
pipeline_state.append({
|
||||
"name": r.name,
|
||||
"status": result,
|
||||
"is_running": False,
|
||||
})
|
||||
except BaseException as e:
|
||||
stack = traceback.format_exception(type(e), e, e.__traceback__)
|
||||
pipeline_state.append({
|
||||
"name": r.name,
|
||||
"status": [str(result), stack],
|
||||
"status": [str(e), stack],
|
||||
"error": True,
|
||||
"is_running": True,
|
||||
})
|
||||
else:
|
||||
pipeline_state.append({
|
||||
"name": r.name,
|
||||
"status": result,
|
||||
"is_running": False,
|
||||
})
|
||||
env_dtos[env.id] = {'id': env.id }
|
||||
env_dtos[env.id]['pipeline'] = pipeline_state
|
||||
try:
|
||||
shared_state = env.state.progress()
|
||||
env_dtos[env.id]['branches'] = shared_state.branches
|
||||
env_dtos[env.id]['branches_token'] = shared_state.token
|
||||
env_dtos[env.id]['dry'] = shared_state.dry
|
||||
except BaseException:
|
||||
pass
|
||||
return env_dtos
|
||||
|
||||
def get_local_branches_to_emit(self) -> Dict[str, Dict[str, List[Any]]]:
|
||||
branches: Dict[str, Dict[str, List[Any]]] = {}
|
||||
for k, env in self.environments.items():
|
||||
branches[k] = {}
|
||||
try:
|
||||
step = get_step(env.pipeline, GitClone)
|
||||
branches[k] = {**step.get_branches()}
|
||||
except BaseException as e:
|
||||
stack = traceback.format_exception(type(e), e, e.__traceback__)
|
||||
pipeline_state.append({
|
||||
"name": r.name,
|
||||
"status": [str(e), stack],
|
||||
"error": True,
|
||||
"is_running": True,
|
||||
})
|
||||
env_dtos[env.id] = asdict(replace(env, pipeline=[]))
|
||||
env_dtos[env.id]['pipeline'] = pipeline_state
|
||||
return env_dtos
|
||||
logger.error(f"Error fetching branches for environment {env.id}: {str(e)}\n{''.join(stack)}")
|
||||
|
||||
|
||||
def get_global_envs_to_emit() -> Any:
|
||||
global environments, environments_slaves
|
||||
local_envs = get_local_envs_to_emit()
|
||||
merge_result = merge_dicts(local_envs, environments_slaves)
|
||||
common_keys = set(local_envs.keys()) & set(environments_slaves.keys())
|
||||
if len(common_keys) > 0:
|
||||
_schedule_async(broadcast_error({'message': f"Conflict: both master and slave have environment with id {common_keys}"}))
|
||||
return merge_result
|
||||
|
||||
|
||||
|
||||
def get_local_branches_to_emit() -> Dict[str, Dict[str, List[Any]]]:
|
||||
global environments
|
||||
branches: Dict[str, Dict[str, List[Any]]] = {}
|
||||
for k, env in environments.items():
|
||||
branches[k] = {}
|
||||
try:
|
||||
for step in env.pipeline:
|
||||
if isinstance(step, GitClone):
|
||||
branches[k] = {**step.get_branches()}
|
||||
if isinstance(step, CachingStep) and isinstance(step.step, GitClone):
|
||||
branches[k] = {**step.step.get_branches()}
|
||||
except BaseException as e:
|
||||
stack = traceback.format_exception(type(e), e, e.__traceback__)
|
||||
logger.error(f"Error fetching branches for environment {env.id}: {str(e)}\n{''.join(stack)}")
|
||||
|
||||
return branches
|
||||
|
||||
|
||||
|
||||
def get_global_branches_to_emit() -> Dict[str, Dict[str, List[Any]]]:
|
||||
global branches_slaves
|
||||
local_branches: Dict[str, Dict[str, List[Any]]] = get_local_branches_to_emit()
|
||||
return merge_dicts(local_branches, branches_slaves)
|
||||
|
||||
|
||||
# --- Async Helper for Calling from Sync Threads ---
|
||||
|
||||
def _schedule_async(coro: Any) -> None:
|
||||
"""Schedule a coroutine from a sync thread using the saved event loop."""
|
||||
if _event_loop is not None and _event_loop.is_running():
|
||||
asyncio.run_coroutine_threadsafe(coro, _event_loop)
|
||||
|
||||
|
||||
# --- WebSocket Broadcasting Functions ---
|
||||
|
||||
async def broadcast(event: str, data: Any) -> None:
|
||||
"""Broadcast a message to all connected WebSocket clients that have not seen this data yet."""
|
||||
disconnected = set()
|
||||
message = custom_json_dumps({event: data})
|
||||
|
||||
for websocket in ws_connections.keys():
|
||||
if ws_connections[websocket].get(event) == message:
|
||||
continue
|
||||
try:
|
||||
await websocket.send_text(message)
|
||||
ws_connections[websocket][event] = message
|
||||
except Exception as e:
|
||||
logger.error(f"Error sending to websocket: {e}")
|
||||
disconnected.add(websocket)
|
||||
|
||||
for ws in disconnected:
|
||||
ws_connections.pop(ws, None)
|
||||
|
||||
|
||||
async def broadcast_branches(data: Any) -> None:
|
||||
await broadcast("branches", data)
|
||||
|
||||
|
||||
async def broadcast_environments(data: Any) -> None:
|
||||
await broadcast("environments", data)
|
||||
|
||||
|
||||
async def broadcast_error(data: Any) -> None:
|
||||
await broadcast("error", data)
|
||||
|
||||
|
||||
# --- Single WebSocket Endpoint ---
|
||||
|
||||
@app.websocket("/ws")
|
||||
async def websocket_endpoint(websocket: WebSocket) -> None:
|
||||
await websocket.accept()
|
||||
ws_connections[websocket] = {}
|
||||
|
||||
try:
|
||||
# Send initial state on connect and record it so duplicate broadcasts are suppressed
|
||||
branches_payload = custom_json_dumps({"branches": get_global_branches_to_emit()})
|
||||
envs_payload = custom_json_dumps({"environments": get_global_envs_to_emit()})
|
||||
await websocket.send_text(branches_payload)
|
||||
ws_connections[websocket]["branches"] = branches_payload
|
||||
await websocket.send_text(envs_payload)
|
||||
ws_connections[websocket]["environments"] = envs_payload
|
||||
|
||||
while True:
|
||||
data = await websocket.receive_text()
|
||||
message = json.loads(data)
|
||||
|
||||
if "update" in message:
|
||||
update_data = message.get("update") or {}
|
||||
logger.info(f"Received environment update: {update_data}")
|
||||
|
||||
if update_data.get('id') == '':
|
||||
reset_caches(list(environments.values()))
|
||||
else:
|
||||
for env in environments.values():
|
||||
if env.id == update_data.get('id'):
|
||||
env.branches = update_data.get('branches', env.branches)
|
||||
if 'dry' in update_data:
|
||||
env.dry = bool(update_data['dry'])
|
||||
logger.info(f"Updated environment {env.id} branches to {env.branches}, dry={env.dry}")
|
||||
reset_caches([env])
|
||||
environment_update_event.set()
|
||||
|
||||
if slave_ws_task is not None:
|
||||
await slave_send_queue.put({"update": update_data})
|
||||
|
||||
await broadcast_environments(get_global_envs_to_emit())
|
||||
|
||||
except WebSocketDisconnect as e:
|
||||
ws_connections.pop(websocket, None)
|
||||
except Exception as e:
|
||||
logger.error(f"WebSocket error: {e}")
|
||||
await broadcast_error({'message': 'WebSocket exception'})
|
||||
ws_connections.pop(websocket, None)
|
||||
|
||||
|
||||
# --- Slave Connection ---
|
||||
|
||||
async def connect_to_slave() -> None:
|
||||
"""Connect to a slave brencher instance via a single WebSocket."""
|
||||
global branches_slaves, environments_slaves
|
||||
|
||||
if not slave_url:
|
||||
return
|
||||
|
||||
logger.info(f"SLAVE_BRENCHER set, will connect to slave at {slave_url}")
|
||||
|
||||
while True:
|
||||
try:
|
||||
ws_url = slave_url.replace('http://', 'ws://').replace('https://', 'wss://') + '/ws'
|
||||
|
||||
async with websockets.connect(ws_url) as ws:
|
||||
logger.info("Connected to slave WebSocket")
|
||||
|
||||
async def handle_messages() -> None:
|
||||
global branches_slaves, environments_slaves
|
||||
async for msg in ws:
|
||||
parsed = json.loads(msg)
|
||||
if "branches" in parsed:
|
||||
branches_slaves = parsed["branches"]
|
||||
await broadcast_branches(get_global_branches_to_emit())
|
||||
elif "environments" in parsed:
|
||||
environments_slaves = parsed["environments"]
|
||||
await broadcast_environments(get_global_envs_to_emit())
|
||||
elif "error" in parsed:
|
||||
await broadcast_error(parsed["error"])
|
||||
|
||||
async def handle_sends() -> None:
|
||||
while True:
|
||||
msg = await slave_send_queue.get()
|
||||
await ws.send(json.dumps(msg))
|
||||
|
||||
await asyncio.gather(
|
||||
handle_messages(),
|
||||
handle_sends(),
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Could not connect to SLAVE_BRENCHER {slave_url}: {e}")
|
||||
await asyncio.sleep(60)
|
||||
|
||||
|
||||
# --- SIGCHLD Handler ---
|
||||
|
||||
def sigchld_handler(signum: int, frame: Any) -> None:
|
||||
"""Reap zombie processes."""
|
||||
while True:
|
||||
try:
|
||||
pid, status = os.waitpid(-1, os.WNOHANG)
|
||||
if pid == 0:
|
||||
break
|
||||
except ChildProcessError:
|
||||
break
|
||||
|
||||
|
||||
# --- Startup Event ---
|
||||
|
||||
@app.on_event("startup")
|
||||
async def startup_event() -> None:
|
||||
global _event_loop, slave_ws_task
|
||||
_event_loop = asyncio.get_event_loop()
|
||||
if slave_url:
|
||||
slave_ws_task = asyncio.create_task(connect_to_slave())
|
||||
|
||||
|
||||
# --- App Class (used by CLI and integration tests) ---
|
||||
|
||||
class App:
|
||||
|
||||
def __init__(self, cli_env_ids_str: str, dry_run: bool = False) -> None:
|
||||
global environments
|
||||
signal.signal(signal.SIGCHLD, sigchld_handler)
|
||||
|
||||
import configs.brencher
|
||||
import configs.brencher2
|
||||
import configs.brencher_local2
|
||||
import configs.brencher_local1
|
||||
import configs.torrserv_proxy
|
||||
import configs.immich
|
||||
import configs.registry
|
||||
import configs.gmail_mcp
|
||||
environments_l: List[Environment] = [
|
||||
configs.brencher.brencher,
|
||||
configs.brencher2.brencher2,
|
||||
configs.brencher_local2.brencher_local2,
|
||||
configs.brencher_local1.brencher_local1,
|
||||
configs.torrserv_proxy.torrserv_proxy,
|
||||
configs.immich.immich,
|
||||
configs.registry.registry,
|
||||
configs.gmail_mcp.gmail_mcp,
|
||||
]
|
||||
environments = {e.id: e for e in environments_l}
|
||||
|
||||
logger.info(f"cli_env_ids {cli_env_ids_str}")
|
||||
if len(cli_env_ids_str) > 0 and cli_env_ids_str[0] == '-':
|
||||
cli_env_ids = cli_env_ids_str[1:].split(',')
|
||||
cli_env_ids = [x for x in cli_env_ids if len(x) > 0]
|
||||
logger.info(f"cli_env_ids (minus) {cli_env_ids}")
|
||||
if cli_env_ids and len(cli_env_ids) > 0:
|
||||
environments = {k: e for k, e in environments.items() if k not in cli_env_ids}
|
||||
else:
|
||||
cli_env_pairs = {x[0]: x[1] if len(x) > 1 else None for x in [x.split(":")
|
||||
for x in cli_env_ids_str.split(',')
|
||||
if len(x) > 0
|
||||
]
|
||||
}
|
||||
logger.info(f"cli_env_ids {cli_env_pairs}")
|
||||
if cli_env_pairs and len(cli_env_pairs) > 0:
|
||||
environments = {k: e for k, e in environments.items() if k in cli_env_pairs.keys()}
|
||||
for k, v in cli_env_pairs.items():
|
||||
if v is not None:
|
||||
if k in environments:
|
||||
env = environments[k]
|
||||
env.branches = [(v, 'HEAD')]
|
||||
logger.info(f"Overriding environment {k} branches to {env.branches}")
|
||||
else:
|
||||
logger.warning(f"Environment {k} not found to override branches")
|
||||
else:
|
||||
logger.info(f"No branch override for environment {k}")
|
||||
if dry_run:
|
||||
for id, e in environments.items():
|
||||
e.dry = True
|
||||
|
||||
logger.info(f"Resulting profiles {environments.keys()}")
|
||||
|
||||
environments = {id: wrap_in_cached(e) for id, e in environments.items()}
|
||||
return branches
|
||||
|
||||
def processing_thread(self) -> None:
|
||||
while True:
|
||||
import processing
|
||||
|
||||
def emit_envs() -> None:
|
||||
_schedule_async(broadcast_branches(get_global_branches_to_emit()))
|
||||
_schedule_async(broadcast_environments(get_global_envs_to_emit()))
|
||||
emit = self.emit_callback or (lambda: None)
|
||||
|
||||
with state_lock:
|
||||
with self.state_lock:
|
||||
logger.info(f"Processing")
|
||||
if processing.process_all_jobs(list(environments.values()), lambda: emit_envs()):
|
||||
environment_update_event.wait(timeout=1 * 5)
|
||||
if processing.process_all_jobs(list(self.environments.values()), emit):
|
||||
self.environment_update_event.wait(timeout=1 * 5)
|
||||
else:
|
||||
environment_update_event.wait(timeout=1 * 60)
|
||||
environment_update_event.clear()
|
||||
self.environment_update_event.wait(timeout=1 * 60)
|
||||
self.environment_update_event.clear()
|
||||
|
||||
def run(self) -> None:
|
||||
processing = threading.Thread(target=self.processing_thread)
|
||||
processing.daemon = True
|
||||
processing.start()
|
||||
|
||||
def runWeb(self, port: int) -> None:
|
||||
self.run()
|
||||
import uvicorn
|
||||
uvicorn.run(app, host='0.0.0.0', port=port, log_level='info')
|
||||
def runHeadless(self) -> None:
|
||||
"""Run the processing loop on the current thread; blocks forever."""
|
||||
self.processing_thread()
|
||||
|
||||
def runWeb(self, port: int) -> None:
|
||||
import web
|
||||
web_app = web.WebApp(core=self, port=port)
|
||||
self.emit_callback = web_app.emit_envs
|
||||
self.run()
|
||||
web_app.start()
|
||||
|
||||
if __name__ == '__main__':
|
||||
cli_env_ids_list = sys.argv[1:]
|
||||
@@ -454,5 +126,15 @@ if __name__ == '__main__':
|
||||
else:
|
||||
cli_env_ids_str = cli_env_ids_list[0]
|
||||
|
||||
app1 = App(cli_env_ids_str, 'dry' in sys.argv[1:])
|
||||
app1.runWeb(port=(5007 if 'noweb' in sys.argv[1:] else 5001))
|
||||
signal.signal(signal.SIGCHLD, sigchld_handler)
|
||||
environments = build_environments(cli_env_ids_str)
|
||||
if 'dry' in sys.argv[1:]:
|
||||
for e in environments.values():
|
||||
e.state.set_dry(True)
|
||||
|
||||
app1 = App(environments)
|
||||
|
||||
if 'noweb' in sys.argv[1:]:
|
||||
app1.runHeadless()
|
||||
else:
|
||||
app1.runWeb(5001)
|
||||
|
||||
@@ -3,9 +3,19 @@ from enironment import Environment
|
||||
from steps.checks import SimpleLog, UrlCheck
|
||||
from steps.docker import DockerComposeBuild, DockerSwarmCheck, DockerSwarmDeploy
|
||||
from steps.git import GitClone, CheckoutMerged, GitUnmerge
|
||||
from steps.shared_state import SharedStateHolderInMemory
|
||||
|
||||
clone = GitClone(url="https://github.com/rudolf1/brencher.git")
|
||||
|
||||
dockerSwarmCheck = DockerSwarmCheck(
|
||||
stack_name="brencher",
|
||||
)
|
||||
unmerge = GitUnmerge(clone, dockerSwarmCheck)
|
||||
|
||||
state = SharedStateHolderInMemory(unmerge=unmerge)
|
||||
|
||||
clone = GitClone()
|
||||
checkoutMerged = CheckoutMerged(clone,
|
||||
desired_branches=state,
|
||||
push=False,
|
||||
git_user_email="rudolfss13@gmail.com",
|
||||
git_user_name="brencher_bot"
|
||||
@@ -24,9 +34,6 @@ buildDocker = DockerComposeBuild(clone,
|
||||
},
|
||||
)
|
||||
|
||||
dockerSwarmCheck = DockerSwarmCheck(
|
||||
stack_name="brencher",
|
||||
)
|
||||
deployDocker = DockerSwarmDeploy(
|
||||
wd=clone,
|
||||
buildDocker=buildDocker,
|
||||
@@ -37,7 +44,6 @@ deployDocker = DockerSwarmDeploy(
|
||||
stack_name="brencher",
|
||||
docker_compose_path="docker-compose.yml",
|
||||
)
|
||||
unmerge = GitUnmerge(clone, dockerSwarmCheck)
|
||||
|
||||
checkPing = UrlCheck(
|
||||
url="https://brencher.rudolf.keenetic.link/state",
|
||||
@@ -53,11 +59,10 @@ logUrls = SimpleLog(message={
|
||||
__all__ = ["brencher"]
|
||||
brencher = Environment(
|
||||
id="brencher",
|
||||
branches=[],
|
||||
dry=False,
|
||||
repo="https://github.com/rudolf1/brencher.git",
|
||||
state=state,
|
||||
pipeline=[
|
||||
clone,
|
||||
state,
|
||||
checkoutMerged,
|
||||
buildDocker,
|
||||
dockerSwarmCheck,
|
||||
|
||||
@@ -5,6 +5,7 @@ from enironment import Environment
|
||||
from steps.checks import SimpleLog, UrlCheck
|
||||
from steps.docker import DockerComposeBuild, DockerSwarmCheck, DockerSwarmDeploy
|
||||
from steps.git import GitClone, CheckoutMerged, GitUnmerge
|
||||
from steps.shared_state import SharedStateHolderInMemory
|
||||
|
||||
|
||||
def checkPingF(obj: Any) -> None:
|
||||
@@ -20,8 +21,18 @@ def checkPingF(obj: Any) -> None:
|
||||
raise Exception(f"brencher2 check failed for: {v}")
|
||||
|
||||
|
||||
clone = GitClone()
|
||||
clone = GitClone(url="https://github.com/rudolf1/brencher.git")
|
||||
|
||||
dockerSwarmCheck = DockerSwarmCheck(
|
||||
stack_name="brencher2",
|
||||
)
|
||||
|
||||
unmerge = GitUnmerge(clone, dockerSwarmCheck)
|
||||
|
||||
state = SharedStateHolderInMemory(unmerge)
|
||||
|
||||
checkoutMerged = CheckoutMerged(clone,
|
||||
desired_branches=state,
|
||||
push=False,
|
||||
git_user_email="rudolfss13@gmail.com",
|
||||
git_user_name="brencher_bot"
|
||||
@@ -40,9 +51,6 @@ buildDocker = DockerComposeBuild(clone,
|
||||
},
|
||||
)
|
||||
|
||||
dockerSwarmCheck = DockerSwarmCheck(
|
||||
stack_name="brencher2",
|
||||
)
|
||||
deployDocker = DockerSwarmDeploy(
|
||||
wd=clone,
|
||||
buildDocker=buildDocker,
|
||||
@@ -63,7 +71,6 @@ deployDocker = DockerSwarmDeploy(
|
||||
stack_name="brencher2",
|
||||
docker_compose_path="docker-compose.yml",
|
||||
)
|
||||
unmerge = GitUnmerge(clone, dockerSwarmCheck)
|
||||
|
||||
checkPing = UrlCheck(
|
||||
url="https://brencher.rudolf.keenetic.link/state",
|
||||
@@ -82,11 +89,10 @@ __all__ = ["brencher2"]
|
||||
|
||||
brencher2 = Environment(
|
||||
id="brencher2",
|
||||
branches=[],
|
||||
dry=False,
|
||||
repo="https://github.com/rudolf1/brencher.git",
|
||||
state=state,
|
||||
pipeline=[
|
||||
clone,
|
||||
state,
|
||||
checkoutMerged,
|
||||
buildDocker,
|
||||
dockerSwarmCheck,
|
||||
@@ -94,5 +100,5 @@ brencher2 = Environment(
|
||||
deployDocker,
|
||||
checkPing,
|
||||
logUrls
|
||||
]
|
||||
],
|
||||
)
|
||||
|
||||
@@ -1,9 +1,20 @@
|
||||
from enironment import Environment
|
||||
from steps.docker_plain import DockerImageBuild, DockerContainerCheck, DockerContainerDeploy
|
||||
from steps.git import GitClone, CheckoutMerged, GitUnmerge
|
||||
from steps.shared_state import SharedStateHolderInMemory
|
||||
|
||||
clone = GitClone(url="https://github.com/rudolf1/brencher.git")
|
||||
|
||||
container_check = DockerContainerCheck(
|
||||
container_name="brencher_plain-container"
|
||||
)
|
||||
|
||||
unmerge = GitUnmerge(wd=clone, check=container_check)
|
||||
|
||||
state = SharedStateHolderInMemory(unmerge=unmerge)
|
||||
|
||||
clone = GitClone()
|
||||
checkoutMerged = CheckoutMerged(clone,
|
||||
desired_branches=state,
|
||||
push=False,
|
||||
git_user_email="rudolfss13@gmail.com",
|
||||
git_user_name="brencher_bot"
|
||||
@@ -18,11 +29,6 @@ image_build = DockerImageBuild(
|
||||
nocache=False,
|
||||
)
|
||||
|
||||
# Step 2: Check if container already exists
|
||||
container_check = DockerContainerCheck(
|
||||
container_name="brencher_plain-container"
|
||||
)
|
||||
|
||||
# Step 3: Deploy the container
|
||||
container_deploy = DockerContainerDeploy(
|
||||
image_build=image_build,
|
||||
@@ -32,16 +38,13 @@ container_deploy = DockerContainerDeploy(
|
||||
restart_policy={"Name": "unless-stopped"},
|
||||
)
|
||||
|
||||
unmerge = GitUnmerge(wd=clone, check=container_check)
|
||||
|
||||
__all__ = ["brencher_local1"]
|
||||
brencher_local1 = Environment(
|
||||
id="brencher_local1",
|
||||
branches=[("main", "HEAD")],
|
||||
dry=False,
|
||||
repo="https://github.com/rudolf1/brencher.git",
|
||||
state=state,
|
||||
pipeline=[
|
||||
clone,
|
||||
state,
|
||||
checkoutMerged,
|
||||
image_build,
|
||||
container_deploy,
|
||||
|
||||
@@ -2,9 +2,18 @@ from enironment import Environment
|
||||
from steps.docker import DockerComposeBuild
|
||||
from steps.docker import DockerSwarmDeploy, DockerSwarmCheck
|
||||
from steps.git import GitClone, CheckoutMerged, GitUnmerge
|
||||
from steps.shared_state import SharedStateHolderInMemory
|
||||
|
||||
clone = GitClone(url="https://github.com/rudolf1/brencher.git")
|
||||
dockerSwarmCheck = DockerSwarmCheck(
|
||||
stack_name="brencher_local2",
|
||||
)
|
||||
unmerge = GitUnmerge(clone, dockerSwarmCheck)
|
||||
|
||||
state = SharedStateHolderInMemory(unmerge=unmerge)
|
||||
|
||||
clone = GitClone()
|
||||
checkoutMerged = CheckoutMerged(clone,
|
||||
desired_branches=state,
|
||||
push=False,
|
||||
git_user_email="rudolfss13@gmail.com",
|
||||
git_user_name="brencher_bot"
|
||||
@@ -22,9 +31,6 @@ buildDocker = DockerComposeBuild(clone,
|
||||
},
|
||||
|
||||
)
|
||||
dockerSwarmCheck = DockerSwarmCheck(
|
||||
stack_name="brencher_local2",
|
||||
)
|
||||
deployDocker = DockerSwarmDeploy(
|
||||
wd=clone,
|
||||
buildDocker=buildDocker,
|
||||
@@ -46,16 +52,14 @@ deployDocker = DockerSwarmDeploy(
|
||||
stack_name="brencher_local2",
|
||||
docker_compose_path="docker-compose.yml",
|
||||
)
|
||||
unmerge = GitUnmerge(clone, dockerSwarmCheck)
|
||||
|
||||
__all__ = ["brencher_local2"]
|
||||
brencher_local2 = Environment(
|
||||
id="brencher_local2",
|
||||
branches=[],
|
||||
dry=False,
|
||||
repo="https://github.com/rudolf1/brencher.git",
|
||||
state=state,
|
||||
pipeline=[
|
||||
clone,
|
||||
state,
|
||||
checkoutMerged,
|
||||
buildDocker,
|
||||
dockerSwarmCheck,
|
||||
|
||||
@@ -2,9 +2,19 @@ from enironment import Environment
|
||||
from steps.checks import SimpleLog, UrlCheck
|
||||
from steps.docker import DockerComposeBuild, DockerSwarmCheck, DockerSwarmDeploy
|
||||
from steps.git import GitClone, CheckoutMerged, GitUnmerge
|
||||
from steps.shared_state import SharedStateHolderInMemory
|
||||
|
||||
clone = GitClone(url="https://github.com/rudolf1/uber_backup.git", branchNamePrefix="immoscout")
|
||||
|
||||
dockerSwarmCheck = DockerSwarmCheck(
|
||||
stack_name="gmail_mcp",
|
||||
)
|
||||
unmerge = GitUnmerge(clone, dockerSwarmCheck)
|
||||
|
||||
state = SharedStateHolderInMemory(unmerge=unmerge)
|
||||
|
||||
clone = GitClone(branchNamePrefix="immoscout")
|
||||
checkoutMerged = CheckoutMerged(clone,
|
||||
desired_branches=state,
|
||||
push=False,
|
||||
git_user_email="rudolfss13@gmail.com",
|
||||
git_user_name="brencher_bot"
|
||||
@@ -21,9 +31,6 @@ buildDocker = DockerComposeBuild(clone,
|
||||
},
|
||||
)
|
||||
|
||||
dockerSwarmCheck = DockerSwarmCheck(
|
||||
stack_name="gmail_mcp",
|
||||
)
|
||||
deployDocker = DockerSwarmDeploy(
|
||||
wd=clone,
|
||||
buildDocker=buildDocker,
|
||||
@@ -34,7 +41,6 @@ deployDocker = DockerSwarmDeploy(
|
||||
stack_name="gmail_mcp",
|
||||
docker_compose_path="gmail-mcp-server/docker-compose.yml",
|
||||
)
|
||||
unmerge = GitUnmerge(clone, dockerSwarmCheck)
|
||||
|
||||
checkPing1 = UrlCheck(
|
||||
url="http://100.70.193.97:3000/health",
|
||||
@@ -55,11 +61,10 @@ logUrls = SimpleLog(message={
|
||||
__all__ = ["gmail_mcp"]
|
||||
gmail_mcp = Environment(
|
||||
id="gmail_mcp",
|
||||
branches=[("immoscout/main", "HEAD")],
|
||||
dry=False,
|
||||
repo="https://github.com/rudolf1/uber_backup.git",
|
||||
state=state,
|
||||
pipeline=[
|
||||
clone,
|
||||
state,
|
||||
checkoutMerged,
|
||||
buildDocker,
|
||||
dockerSwarmCheck,
|
||||
|
||||
+14
-10
@@ -2,17 +2,23 @@ from enironment import Environment
|
||||
from steps.checks import SimpleLog, UrlCheck
|
||||
from steps.docker import DockerSwarmCheck, DockerSwarmDeploy
|
||||
from steps.git import GitClone, CheckoutMerged, GitUnmerge
|
||||
from steps.shared_state import SharedStateHolderInMemory
|
||||
|
||||
clone = GitClone(branchNamePrefix="ansible")
|
||||
checkoutMerged = CheckoutMerged(clone,
|
||||
push=False,
|
||||
git_user_email="rudolfss13@gmail.com",
|
||||
git_user_name="brencher_bot"
|
||||
)
|
||||
clone = GitClone(url="https://github.com/rudolf1/uber_backup.git", branchNamePrefix="ansible")
|
||||
|
||||
dockerSwarmCheck = DockerSwarmCheck(
|
||||
stack_name="immich",
|
||||
)
|
||||
unmerge = GitUnmerge(clone, dockerSwarmCheck)
|
||||
|
||||
state = SharedStateHolderInMemory(unmerge=unmerge)
|
||||
|
||||
checkoutMerged = CheckoutMerged(clone,
|
||||
desired_branches=state,
|
||||
push=False,
|
||||
git_user_email="rudolfss13@gmail.com",
|
||||
git_user_name="brencher_bot"
|
||||
)
|
||||
|
||||
deployDocker = DockerSwarmDeploy(
|
||||
wd=clone,
|
||||
@@ -24,7 +30,6 @@ deployDocker = DockerSwarmDeploy(
|
||||
stack_name="immich",
|
||||
docker_compose_path="poc/immich/stack-compose.yml",
|
||||
)
|
||||
unmerge = GitUnmerge(clone, dockerSwarmCheck)
|
||||
|
||||
checkPing = UrlCheck(
|
||||
url="https://immich.rudolf.keenetic.link/api/server/ping",
|
||||
@@ -39,11 +44,10 @@ logUrls = SimpleLog(message={
|
||||
__all__ = ["immich"]
|
||||
immich = Environment(
|
||||
id="immich",
|
||||
branches=[("ansible/master", "HEAD")],
|
||||
dry=False,
|
||||
repo="https://github.com/rudolf1/uber_backup.git",
|
||||
state=state,
|
||||
pipeline=[
|
||||
clone,
|
||||
state,
|
||||
checkoutMerged,
|
||||
dockerSwarmCheck,
|
||||
unmerge,
|
||||
|
||||
+14
-10
@@ -2,17 +2,23 @@ from enironment import Environment
|
||||
from steps.checks import SimpleLog, UrlCheck
|
||||
from steps.docker import DockerSwarmCheck, DockerSwarmDeploy
|
||||
from steps.git import GitClone, CheckoutMerged, GitUnmerge
|
||||
from steps.shared_state import SharedStateHolderInMemory
|
||||
|
||||
clone = GitClone(branchNamePrefix="ansible")
|
||||
checkoutMerged = CheckoutMerged(clone,
|
||||
push=False,
|
||||
git_user_email="rudolfss13@gmail.com",
|
||||
git_user_name="brencher_bot"
|
||||
)
|
||||
clone = GitClone(url="https://github.com/rudolf1/uber_backup.git", branchNamePrefix="ansible")
|
||||
|
||||
dockerSwarmCheck = DockerSwarmCheck(
|
||||
stack_name="registry",
|
||||
)
|
||||
unmerge = GitUnmerge(clone, dockerSwarmCheck)
|
||||
|
||||
state = SharedStateHolderInMemory(unmerge=unmerge)
|
||||
|
||||
checkoutMerged = CheckoutMerged(clone,
|
||||
desired_branches=state,
|
||||
push=False,
|
||||
git_user_email="rudolfss13@gmail.com",
|
||||
git_user_name="brencher_bot"
|
||||
)
|
||||
|
||||
deployDocker = DockerSwarmDeploy(
|
||||
wd=clone,
|
||||
@@ -24,7 +30,6 @@ deployDocker = DockerSwarmDeploy(
|
||||
stack_name="registry",
|
||||
docker_compose_path="docker-compose-registry.yaml",
|
||||
)
|
||||
unmerge = GitUnmerge(clone, dockerSwarmCheck)
|
||||
|
||||
checkPing = UrlCheck(
|
||||
url="https://registry.rudolf.keenetic.link/v2/",
|
||||
@@ -39,11 +44,10 @@ logUrls = SimpleLog(message={
|
||||
__all__ = ["registry"]
|
||||
registry = Environment(
|
||||
id="registry",
|
||||
branches=[("ansible/master", "HEAD")],
|
||||
dry=False,
|
||||
repo="https://github.com/rudolf1/uber_backup.git",
|
||||
state=state,
|
||||
pipeline=[
|
||||
clone,
|
||||
state,
|
||||
checkoutMerged,
|
||||
dockerSwarmCheck,
|
||||
unmerge,
|
||||
|
||||
@@ -1,9 +1,19 @@
|
||||
from enironment import Environment
|
||||
from steps.docker import DockerComposeBuild, DockerSwarmCheck, DockerSwarmDeploy
|
||||
from steps.git import GitClone, CheckoutMerged, GitUnmerge
|
||||
from steps.shared_state import SharedStateHolderInMemory
|
||||
|
||||
clone = GitClone(url="https://github.com/rudolf1/reverseproxy.git")
|
||||
|
||||
dockerSwarmCheck = DockerSwarmCheck(
|
||||
stack_name="torrserv_proxy",
|
||||
)
|
||||
unmerge = GitUnmerge(clone, dockerSwarmCheck)
|
||||
|
||||
state = SharedStateHolderInMemory(unmerge=unmerge)
|
||||
|
||||
clone = GitClone()
|
||||
checkoutMerged = CheckoutMerged(clone,
|
||||
desired_branches=state,
|
||||
push=False,
|
||||
git_user_email="rudolfss13@gmail.com",
|
||||
git_user_name="brencher_bot"
|
||||
@@ -20,9 +30,6 @@ buildDocker = DockerComposeBuild(clone,
|
||||
},
|
||||
)
|
||||
|
||||
dockerSwarmCheck = DockerSwarmCheck(
|
||||
stack_name="torrserv_proxy",
|
||||
)
|
||||
deployDocker = DockerSwarmDeploy(
|
||||
wd=clone,
|
||||
buildDocker=buildDocker,
|
||||
@@ -33,16 +40,14 @@ deployDocker = DockerSwarmDeploy(
|
||||
stack_name="torrserv_proxy",
|
||||
docker_compose_path="docker-compose.yml",
|
||||
)
|
||||
unmerge = GitUnmerge(clone, dockerSwarmCheck)
|
||||
|
||||
__all__ = ["torrserv_proxy"]
|
||||
torrserv_proxy = Environment(
|
||||
id="torrserv_proxy",
|
||||
branches=[],
|
||||
dry=False,
|
||||
repo="https://github.com/rudolf1/reverseproxy.git",
|
||||
state=state,
|
||||
pipeline=[
|
||||
clone,
|
||||
state,
|
||||
checkoutMerged,
|
||||
buildDocker,
|
||||
dockerSwarmCheck,
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
import importlib
|
||||
import logging
|
||||
import pkgutil
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from enironment import Environment, SharedStateHolder
|
||||
from steps.step import CachingStep
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@dataclass
|
||||
class ParsedEnvArgs:
|
||||
excludes: List[str] = field(default_factory=list)
|
||||
includes: Dict[str, Optional[str]] = field(default_factory=dict)
|
||||
|
||||
|
||||
def parse_arguments(cli_env_ids_str: str) -> ParsedEnvArgs:
|
||||
logger.info(f"cli_env_ids {cli_env_ids_str}")
|
||||
if len(cli_env_ids_str) > 0 and cli_env_ids_str[0] == '-':
|
||||
excludes = [x for x in cli_env_ids_str[1:].split(',') if len(x) > 0]
|
||||
logger.info(f"cli_env_ids (minus) {excludes}")
|
||||
return ParsedEnvArgs(excludes=excludes)
|
||||
|
||||
includes: Dict[str, Optional[str]] = {
|
||||
x[0]: x[1] if len(x) > 1 else None
|
||||
for x in [x.split(":") for x in cli_env_ids_str.split(',') if len(x) > 0]
|
||||
}
|
||||
logger.info(f"cli_env_ids {includes}")
|
||||
return ParsedEnvArgs(includes=includes)
|
||||
|
||||
|
||||
def _discover_environments() -> Dict[str, Environment]:
|
||||
import configs
|
||||
found: Dict[str, Environment] = {}
|
||||
seen: set[int] = set()
|
||||
for module_info in pkgutil.iter_modules(configs.__path__, prefix=f"{configs.__name__}."):
|
||||
module = importlib.import_module(module_info.name)
|
||||
for value in vars(module).values():
|
||||
if isinstance(value, Environment) and id(value) not in seen:
|
||||
seen.add(id(value))
|
||||
found[value.id] = value
|
||||
return found
|
||||
|
||||
|
||||
def build_environments(cli_env_ids_str: str) -> Dict[str, Environment]:
|
||||
environments: Dict[str, Environment] = _discover_environments()
|
||||
args: ParsedEnvArgs = parse_arguments(cli_env_ids_str)
|
||||
|
||||
if args.excludes:
|
||||
environments = {k: e for k, e in environments.items() if k not in args.excludes}
|
||||
elif args.includes:
|
||||
environments = {k: e for k, e in environments.items() if k in args.includes}
|
||||
for k, v in args.includes.items():
|
||||
if v is None:
|
||||
logger.info(f"No branch override for environment {k}")
|
||||
continue
|
||||
if k not in environments:
|
||||
logger.warning(f"Environment {k} not found to override branches")
|
||||
continue
|
||||
env = environments[k]
|
||||
for step in env.pipeline:
|
||||
resolve_step = step
|
||||
if isinstance(step, CachingStep):
|
||||
resolve_step = step._step
|
||||
if isinstance(resolve_step, SharedStateHolder):
|
||||
resolve_step.set_branches([(v, 'HEAD')])
|
||||
logger.info(f"Overriding environment {k} branches to {(v, 'HEAD')}")
|
||||
|
||||
logger.info(f"Resulting profiles {environments.keys()}")
|
||||
return environments
|
||||
+45
-9
@@ -3,7 +3,7 @@ from __future__ import annotations
|
||||
import logging
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from typing import List, Tuple
|
||||
from typing import List, runtime_checkable, Protocol, Tuple, TypeAlias, Any
|
||||
from typing import TypeVar
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -14,15 +14,43 @@ T = TypeVar('T')
|
||||
@dataclass
|
||||
class Environment:
|
||||
id: str
|
||||
branches: List[Tuple[str, str]] # List of [branch_name, desired_commit] pairs
|
||||
dry: bool
|
||||
repo: str # git repo
|
||||
state: SharedStateHolder
|
||||
pipeline: List[AbstractStep]
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
for p in self.pipeline:
|
||||
p.env = self
|
||||
|
||||
@property
|
||||
def dry(self) -> bool:
|
||||
"""Get the dry run state from the shared state."""
|
||||
return self.state.progress().dry
|
||||
|
||||
|
||||
|
||||
@dataclass
|
||||
class SharedState:
|
||||
branches: List[Tuple[str, str]]
|
||||
dry: bool
|
||||
token: str | None = None
|
||||
|
||||
|
||||
class SharedStateConflictError(BaseException):
|
||||
def __init__(self, message: str) -> None:
|
||||
super().__init__(message)
|
||||
|
||||
@runtime_checkable
|
||||
class SharedStateHolder(Protocol):
|
||||
def set_branches(self, branches: List[Tuple[str, str]], expected_token: str | None = None) -> SharedState:
|
||||
pass
|
||||
|
||||
def set_dry(self, dry: bool, expected_token: str | None = None) -> SharedState:
|
||||
pass
|
||||
|
||||
def progress(self) -> SharedState:
|
||||
pass
|
||||
|
||||
|
||||
|
||||
class AbstractStep[T](ABC):
|
||||
_env: Environment | None = None
|
||||
@@ -55,16 +83,24 @@ def wrap_in_cached(e: Environment) -> Environment:
|
||||
from dataclasses import replace
|
||||
from steps.step import CachingStep
|
||||
result = replace(e, pipeline=[CachingStep(step) for step in e.pipeline])
|
||||
cached = {it.step: it for it in result.pipeline if isinstance(it, CachingStep)}
|
||||
cached = {it._step: it for it in result.pipeline if isinstance(it, CachingStep)}
|
||||
for step in result.pipeline:
|
||||
if isinstance(step, CachingStep):
|
||||
step.step.env = result
|
||||
for attr_name in dir(step.step):
|
||||
step._step.env = result
|
||||
for attr_name in dir(step._step):
|
||||
if not attr_name.startswith('_'):
|
||||
attr_value = getattr(step.step, attr_name)
|
||||
attr_value = getattr(step._step, attr_name)
|
||||
if isinstance(attr_value, AbstractStep):
|
||||
cached_step = cached.get(attr_value)
|
||||
if cached_step:
|
||||
setattr(step.step, attr_name, cached_step)
|
||||
setattr(step._step, attr_name, cached_step)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def get_step(env: List[AbstractStep[Any]], class_or_tuple: type[T]) -> T:
|
||||
from steps.step import CachingStep
|
||||
for step in env:
|
||||
if isinstance(step, class_or_tuple) or (isinstance(step, CachingStep) and isinstance(step._step, class_or_tuple)):
|
||||
return step # type: ignore[return-value]
|
||||
raise BaseException(f"Not step found {class_or_tuple}")
|
||||
@@ -3,7 +3,6 @@ import time
|
||||
from typing import List, Callable
|
||||
|
||||
from enironment import Environment
|
||||
from steps.git import GitUnmerge, GitUnmergeResult
|
||||
from steps.step import CachingStep
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -35,20 +34,6 @@ def process_all_jobs(
|
||||
try:
|
||||
onupdate()
|
||||
step.progress()
|
||||
if (isinstance(step, GitUnmerge) or (
|
||||
isinstance(step, CachingStep) and isinstance(step.step, GitUnmerge))) and len(
|
||||
env.branches) == 0:
|
||||
# TODO Move to separate job.
|
||||
# If branches list empty, need to find any brunch which includes commit and add pair (branch, commit)
|
||||
# If branches not empty, need to find most priority branch (project specific) and add (branch, HEAD)
|
||||
result = step.progress()
|
||||
if isinstance(result, GitUnmergeResult):
|
||||
env.branches = result.branches
|
||||
else:
|
||||
# Backward compatibility: older or custom step implementations may
|
||||
# return a plain list of (branch, commit) tuples directly.
|
||||
env.branches = result
|
||||
logger.info(f"Branches on startup resolved {env.id}, job {step.name}: {env.branches}")
|
||||
except BaseException as e:
|
||||
error_msg = f"Error processing release {env.id}, job {step.name}: {str(e)}"
|
||||
logger.error(error_msg)
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from typing import Any, Awaitable, Callable, Dict, List, Optional, Iterator
|
||||
|
||||
import websockets
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SecondaryConnector:
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
url: str,
|
||||
on_update: Callable[[], Awaitable[None]],
|
||||
) -> None:
|
||||
self._url = url
|
||||
self.branches: Dict[str, Any] = {}
|
||||
self.environments: Dict[str, Any] = {}
|
||||
self._task: Optional[asyncio.Task[None]] = None
|
||||
self._on_update = on_update
|
||||
self._ws: Optional[Any] = None
|
||||
|
||||
def start(self) -> None:
|
||||
self._task = asyncio.create_task(self._connect())
|
||||
|
||||
@property
|
||||
def is_active(self) -> bool:
|
||||
return self._task is not None and not self._task.done()
|
||||
|
||||
async def send(self, msg: Dict[str, Any]) -> None:
|
||||
|
||||
payload = json.dumps(msg)
|
||||
max_attempts = 3
|
||||
timeout_seconds = 10.0
|
||||
|
||||
for attempt in range(1, max_attempts + 1):
|
||||
if self._ws:
|
||||
try:
|
||||
await asyncio.wait_for(self._ws.send(payload), timeout=timeout_seconds)
|
||||
return
|
||||
except Exception as e:
|
||||
logger.warning(f"Send attempt {attempt}/{max_attempts} failed for {self._url}: {e}")
|
||||
|
||||
if attempt < max_attempts:
|
||||
await asyncio.sleep(timeout_seconds * attempt)
|
||||
|
||||
logger.error(f"Dropping message for secondary {self._url} after {max_attempts} attempts")
|
||||
|
||||
async def _connect(self) -> None:
|
||||
logger.info(f"Connecting to secondary at {self._url}")
|
||||
reconnect_delay_seconds = 5
|
||||
|
||||
while True:
|
||||
try:
|
||||
ws_url = (
|
||||
self._url
|
||||
.replace('http://', 'ws://')
|
||||
.replace('https://', 'wss://')
|
||||
+ '/ws'
|
||||
)
|
||||
|
||||
async with websockets.connect(
|
||||
ws_url,
|
||||
ping_interval=20,
|
||||
ping_timeout=20,
|
||||
close_timeout=5,
|
||||
) as ws:
|
||||
self._ws = ws
|
||||
logger.info(f"Connected to secondary WebSocket at {self._url}")
|
||||
reconnect_delay_seconds = 5
|
||||
|
||||
async for msg in ws:
|
||||
parsed = json.loads(msg)
|
||||
if "branches" in parsed:
|
||||
self.branches = parsed["branches"]
|
||||
await self._on_update()
|
||||
elif "environments" in parsed:
|
||||
self.environments = parsed["environments"]
|
||||
await self._on_update()
|
||||
elif "error" in parsed:
|
||||
await self._on_update()
|
||||
|
||||
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
logger.exception(
|
||||
f"Unexpected secondary connector error for {self._url}. "
|
||||
f"Retrying in {reconnect_delay_seconds}s"
|
||||
)
|
||||
finally:
|
||||
self._ws = None
|
||||
|
||||
await asyncio.sleep(reconnect_delay_seconds)
|
||||
reconnect_delay_seconds = min(reconnect_delay_seconds * 2, 60)
|
||||
|
||||
class SecondaryManager():
|
||||
def __init__(self,
|
||||
urls: str,
|
||||
on_update: Callable[[], Awaitable[None]],
|
||||
) -> None:
|
||||
self.lst: List[SecondaryConnector] = []
|
||||
urls_list = [it.strip() for it in urls.split(",") if it.strip()]
|
||||
logger.info(f"SECONDARY_BRENCHER is {urls_list}")
|
||||
for url in urls_list:
|
||||
connector = SecondaryConnector(
|
||||
url,
|
||||
on_update=on_update,
|
||||
)
|
||||
connector.start()
|
||||
self.lst.append(connector)
|
||||
|
||||
def __iter__(self) -> Iterator[SecondaryConnector]:
|
||||
return iter(self.lst)
|
||||
|
||||
async def send(self, msg: Dict[str, Any]) -> None:
|
||||
await asyncio.gather(*(connector.send(msg) for connector in self.lst), return_exceptions=True)
|
||||
@@ -279,8 +279,7 @@ class DockerSwarmDeploy(AbstractStep[str]):
|
||||
swarmEnv = {k: v for k, v in dotenv_values(os.path.join(os.path.dirname(tmp_compose_path), ".env")).items()
|
||||
if v is not None}
|
||||
# merge_dicts(swarmEnv, env)
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
# , cwd=os.path.dirname(tmp_compose_path), env=swarmEnv)
|
||||
result = subprocess.run(cmd, capture_output=True, text=True, cwd=os.path.dirname(tmp_compose_path), env=swarmEnv)
|
||||
os.remove(tmp_compose_path)
|
||||
if result.returncode != 0:
|
||||
logger.error(f"Stack deploy failed: {result.stderr}")
|
||||
|
||||
@@ -71,6 +71,10 @@ class DockerImageBuild(AbstractStep[DockerImageBuildResult]):
|
||||
if len(existing_image) > 0 and not self.nocache:
|
||||
logger.info(f"Image {full_image} already exists locally with ID: {existing_image}")
|
||||
build_logs: List[str] = []
|
||||
elif self.env.dry:
|
||||
logger.info(f"DRY RUN: Would build image {full_image}")
|
||||
raise BaseException(f"Dry run - skipping actual build for {full_image}")
|
||||
|
||||
else:
|
||||
# Build the image
|
||||
build_logs = []
|
||||
|
||||
+23
-16
@@ -8,16 +8,19 @@ import traceback
|
||||
from typing import List, Tuple, Set, Dict, Any, Mapping, runtime_checkable
|
||||
|
||||
import git
|
||||
from enironment import AbstractStep
|
||||
from enironment import AbstractStep, SharedState
|
||||
from git.objects import Commit
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class GitClone(AbstractStep[str]):
|
||||
def __init__(self, repo_path: str | None = None, branchNamePrefix: str = "", credEnvPrefix: str = "GIT",
|
||||
def __init__(self, url: str, repo_path: str | None = None,
|
||||
branchNamePrefix: str = "", credEnvPrefix: str = "GIT",
|
||||
**kwargs: Any):
|
||||
super().__init__(**kwargs)
|
||||
self.url = url
|
||||
self.repo_path = repo_path
|
||||
self.branchNamePrefix = branchNamePrefix
|
||||
self.credEnvPrefix = credEnvPrefix
|
||||
@@ -25,7 +28,7 @@ class GitClone(AbstractStep[str]):
|
||||
def _get_auth_git_url(self, url: str) -> str:
|
||||
username = os.getenv(f'{self.credEnvPrefix}_USERNAME')
|
||||
password = os.getenv(f'{self.credEnvPrefix}_PASSWORD')
|
||||
if username and password:
|
||||
if username and password and '://' in url:
|
||||
# Extract protocol and the rest of the URL
|
||||
protocol, rest = url.split('://')
|
||||
return f"{protocol}://{username}:{password}@{rest}"
|
||||
@@ -33,32 +36,33 @@ class GitClone(AbstractStep[str]):
|
||||
return url
|
||||
|
||||
def progress(self) -> str:
|
||||
repo_url = self.url
|
||||
|
||||
self.repo_path = self.repo_path or os.path.join(tempfile.gettempdir(),
|
||||
f"{self.env.id}_{hashlib.sha1(self.env.repo.encode()).hexdigest()[:5]}")
|
||||
logger.info(f"Cloning repository {self.env.repo} to {self.repo_path}")
|
||||
f"{self.env.id}_{hashlib.sha1(repo_url.encode()).hexdigest()[:5]}")
|
||||
logger.info(f"Cloning repository {repo_url} to {self.repo_path}")
|
||||
os.makedirs(self.repo_path, exist_ok=True)
|
||||
try:
|
||||
if os.path.exists(os.path.join(self.repo_path, ".git")):
|
||||
logger.info(f"Repository already cloned at {self.repo_path}, fetching updates.")
|
||||
repo = git.Repo(self.repo_path)
|
||||
if repo.remotes.origin.url != self._get_auth_git_url(self.env.repo):
|
||||
repo.remotes.origin.set_url(self._get_auth_git_url(self.env.repo))
|
||||
if 'origin' not in repo.remotes or repo.remotes.origin.url != self._get_auth_git_url(repo_url):
|
||||
repo.create_remote('origin', self._get_auth_git_url(repo_url))
|
||||
result = repo.remotes.origin.fetch(prune=True)
|
||||
if not result or any(fetch_info.flags & fetch_info.ERROR for fetch_info in result):
|
||||
raise BaseException(f"Failed to fetch updates for {self.env.repo}")
|
||||
raise BaseException(f"Failed to fetch updates for {repo_url}")
|
||||
else:
|
||||
repo = git.Repo.init(self.repo_path)
|
||||
repo.remotes.append(repo.create_remote(
|
||||
'origin',
|
||||
self._get_auth_git_url(self.env.repo)
|
||||
self._get_auth_git_url(repo_url)
|
||||
))
|
||||
if self.branchNamePrefix != "":
|
||||
repo.config_writer().set_value('remote "origin"', "fetch",
|
||||
f"+refs/heads/{self.branchNamePrefix}/*:refs/remotes/origin/{self.branchNamePrefix}/*").release()
|
||||
repo.remotes.origin.fetch(prune=True)
|
||||
if not os.path.exists(os.path.join(self.repo_path, ".git")):
|
||||
raise BaseException(f"Failed to clone repository {self.env.repo} to {self.repo_path}")
|
||||
raise BaseException(f"Failed to clone repository {repo_url} to {self.repo_path}")
|
||||
except BaseException as e:
|
||||
logger.error(f"Error during git clone/fetch, removing directory {self.repo_path}: {str(e)}")
|
||||
shutil.rmtree(self.repo_path)
|
||||
@@ -112,12 +116,14 @@ class CheckoutMerged(AbstractStep[CheckoutAndMergeResult]):
|
||||
wd: GitClone
|
||||
|
||||
def __init__(self, wd: GitClone,
|
||||
desired_branches: AbstractStep[SharedState],
|
||||
git_user_email: str,
|
||||
git_user_name: str,
|
||||
push: bool = True,
|
||||
**kwargs: Any):
|
||||
super().__init__(**kwargs)
|
||||
self.wd = wd
|
||||
self.desired_branches = desired_branches
|
||||
self.git_user_email = git_user_email
|
||||
self.git_user_name = git_user_name
|
||||
self.push = push
|
||||
@@ -137,10 +143,10 @@ class CheckoutMerged(AbstractStep[CheckoutAndMergeResult]):
|
||||
|
||||
return visited
|
||||
|
||||
def find_desired_commits(self, repo: git.Repo) -> Dict[Commit, str]:
|
||||
def find_desired_commits(self, repo: git.Repo, branches: List[Tuple[str, str]]) -> Dict[Commit, str]:
|
||||
# Extract commit ids for the selected branches
|
||||
commit_ids: Dict[Commit, str] = {}
|
||||
for branch_pair in self.env.branches:
|
||||
for branch_pair in branches:
|
||||
try:
|
||||
branch_name, desired_commit = branch_pair
|
||||
if desired_commit == 'HEAD':
|
||||
@@ -164,12 +170,13 @@ class CheckoutMerged(AbstractStep[CheckoutAndMergeResult]):
|
||||
cw.set_value("user", "email", self.git_user_email)
|
||||
cw.set_value("user", "name", self.git_user_name)
|
||||
|
||||
if len(self.env.branches) == 0:
|
||||
desired = self.desired_branches.progress().branches
|
||||
if len(desired) == 0:
|
||||
raise BaseException(f"Empty branches set")
|
||||
|
||||
logger.info(f"Selected branches: {self.env.branches}")
|
||||
logger.info(f"Selected branches: {desired}")
|
||||
|
||||
commit_ids = self.find_desired_commits(repo)
|
||||
commit_ids = self.find_desired_commits(repo, desired)
|
||||
logger.info(f"Commit ids for branches: {commit_ids}")
|
||||
|
||||
childs = _commits_childs(repo)
|
||||
@@ -293,7 +300,7 @@ class GitUnmerge(AbstractStep[GitUnmergeResult]):
|
||||
if len(branches) == 0 or branches[0].startswith('auto/'):
|
||||
if childs is None:
|
||||
childs = _commits_childs(repo)
|
||||
commitsSet = set([commit])
|
||||
commitsSet = {commit}
|
||||
while len(commitsSet) > 0:
|
||||
current = commitsSet.pop()
|
||||
for child in childs.get(current, []):
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import tempfile
|
||||
import threading
|
||||
import uuid
|
||||
from dataclasses import replace
|
||||
from typing import List, Tuple, Any
|
||||
|
||||
import git
|
||||
|
||||
from enironment import AbstractStep, SharedState, SharedStateHolder, SharedStateConflictError
|
||||
from steps.git import GitUnmergeResult, GitClone
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _normalize_branches(branches: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
|
||||
return sorted(branches, key=lambda x: (x[0], x[1]))
|
||||
|
||||
|
||||
class SharedStateHolderInMemory(AbstractStep[SharedState], SharedStateHolder):
|
||||
|
||||
def __init__(self, unmerge: AbstractStep[GitUnmergeResult] | None, **kwargs: Any) -> None:
|
||||
super().__init__(**kwargs)
|
||||
self.state = SharedState([], False, token=uuid.uuid4().hex)
|
||||
self.unmerge = unmerge
|
||||
|
||||
def set_branches(self, branches: List[Tuple[str, str]], expected_token: str | None = None) -> SharedState:
|
||||
if expected_token is not None and expected_token != self.state.token:
|
||||
raise SharedStateConflictError("State token mismatch")
|
||||
self.state = replace(self.state, branches=_normalize_branches(branches), token=uuid.uuid4().hex)
|
||||
return self.state
|
||||
|
||||
def set_dry(self, dry: bool, expected_token: str | None = None) -> SharedState:
|
||||
if expected_token is not None and expected_token != self.state.token:
|
||||
raise SharedStateConflictError("State token mismatch")
|
||||
self.state = replace(self.state, dry=dry, token=uuid.uuid4().hex)
|
||||
return self.state
|
||||
|
||||
def progress(self) -> SharedState:
|
||||
if len(self.state.branches) == 0:
|
||||
if self.unmerge:
|
||||
unmerge = self.unmerge.progress()
|
||||
self.state = replace(self.state, branches=unmerge.branches, token=uuid.uuid4().hex)
|
||||
return self.state
|
||||
|
||||
|
||||
class SharedStateHolderInGit(AbstractStep[SharedState], SharedStateHolder):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
wd: GitClone,
|
||||
state_repo: GitClone,
|
||||
state_branch: str = "state",
|
||||
**kwargs: Any
|
||||
) -> None:
|
||||
super().__init__(**kwargs)
|
||||
self.state_branch = state_branch
|
||||
self.wd = wd
|
||||
self.state_repo = state_repo
|
||||
self._lock = threading.RLock()
|
||||
|
||||
def set_branches(self, branches: List[Tuple[str, str]], expected_token: str | None = None) -> SharedState:
|
||||
with self._lock:
|
||||
repo = self._ensure_repo(expected_token)
|
||||
oldState = self._read(repo)
|
||||
if expected_token is not None and expected_token != oldState.token:
|
||||
raise SharedStateConflictError("State token mismatch")
|
||||
|
||||
newState = replace(oldState, branches=_normalize_branches(branches))
|
||||
self._write(repo, newState)
|
||||
return self._read(repo)
|
||||
|
||||
def set_dry(self, dry: bool, expected_token: str | None = None) -> SharedState:
|
||||
with self._lock:
|
||||
repo = self._ensure_repo(expected_token)
|
||||
oldState = self._read(repo)
|
||||
newState = replace(oldState, dry=dry)
|
||||
self._write(repo, newState)
|
||||
return self._read(repo)
|
||||
|
||||
def progress(self) -> SharedState:
|
||||
with self._lock:
|
||||
repo = self._ensure_repo(None)
|
||||
return self._read(repo)
|
||||
|
||||
def file_path(self, repo: git.Repo) -> str:
|
||||
return f"{repo.working_dir}/{self.env.id}.json"
|
||||
|
||||
def _ensure_repo(self, token: str | None = None) -> git.Repo:
|
||||
repo_path = self.state_repo.progress()
|
||||
repo = git.Repo(repo_path)
|
||||
if token is None:
|
||||
# Checkout state branch in detached HEAD state
|
||||
token = f"origin/{self.state_branch}"
|
||||
remote_refs = {ref.name for ref in repo.refs}
|
||||
if not token in remote_refs:
|
||||
raise BaseException("Branch not found")
|
||||
|
||||
repo.git.checkout("--detach", token)
|
||||
return repo
|
||||
|
||||
def _read(self, repo: git.Repo) -> SharedState:
|
||||
state_abs_path = self.file_path(repo)
|
||||
|
||||
os.makedirs(os.path.dirname(state_abs_path), exist_ok=True)
|
||||
|
||||
state_data: dict[str, Any] = {}
|
||||
if os.path.exists(state_abs_path):
|
||||
try:
|
||||
with open(state_abs_path) as f:
|
||||
state_data = json.load(f)
|
||||
except json.JSONDecodeError as e:
|
||||
raise Exception(f"Invalid JSON in state file {state_abs_path}: {str(e)}") from e
|
||||
|
||||
branches: List[Tuple[str, str]] = []
|
||||
if isinstance(state_data.get("branches", []), list):
|
||||
for item in state_data.get("branches", []):
|
||||
if isinstance(item, (list, tuple)) and len(item) == 2:
|
||||
branches.append((str(item[0]), str(item[1])))
|
||||
dry = bool(state_data.get("dry", True))
|
||||
return SharedState(branches=branches, dry=dry, token=repo.head.commit.hexsha)
|
||||
|
||||
def _write(self, repo: git.Repo, state: SharedState) -> None:
|
||||
state_abs_path = self.file_path(repo)
|
||||
os.makedirs(os.path.dirname(state_abs_path), exist_ok=True)
|
||||
state_data = {
|
||||
"branches": state.branches,
|
||||
"dry": state.dry,
|
||||
}
|
||||
|
||||
new_state_content = json.dumps(state_data, sort_keys=True, indent=2) + "\n"
|
||||
with open(state_abs_path, "w") as f:
|
||||
f.write(new_state_content)
|
||||
repo.index.add([state_abs_path])
|
||||
repo.index.commit(f"Update state for {self.env.id}")
|
||||
try:
|
||||
remote_state_ref = f"origin/{self.state_branch}"
|
||||
repo.git.merge(remote_state_ref, no_edit=True)
|
||||
repo.git.push("origin", f"HEAD:refs/heads/{self.state_branch}")
|
||||
except git.GitCommandError as e:
|
||||
try:
|
||||
repo.git.merge("--abort")
|
||||
except git.GitCommandError:
|
||||
pass
|
||||
raise SharedStateConflictError("State token mismatch")
|
||||
|
||||
@@ -27,14 +27,14 @@ class CachingStep(AbstractStep[T], Generic[T]):
|
||||
|
||||
def __init__(self, step: AbstractStep[T]) -> None:
|
||||
super().__init__(n=step.name)
|
||||
self.step = step
|
||||
self._result = NotReadyException(f"No result yet for {self.step.name}")
|
||||
self._step = step
|
||||
self._result = NotReadyException(f"No result yet for {self._step.name}")
|
||||
self._input_hash = None
|
||||
|
||||
def _compute_input_hash(self) -> str:
|
||||
"""Compute a stable hash of all AbstractStep dependency outputs."""
|
||||
inputs: dict[str, Any] = {}
|
||||
for attr_name, attr_value in vars(self.step).items():
|
||||
for attr_name, attr_value in vars(self._step).items():
|
||||
if isinstance(attr_value, AbstractStep):
|
||||
try:
|
||||
inputs[attr_name] = attr_value.progress()
|
||||
@@ -46,7 +46,7 @@ class CachingStep(AbstractStep[T], Generic[T]):
|
||||
current_hash = self._compute_input_hash()
|
||||
if self._input_hash != current_hash or isinstance(self._result, BaseException):
|
||||
try:
|
||||
self._result = self.step.progress()
|
||||
self._result = self._step.progress()
|
||||
except BaseException as e:
|
||||
self._result = e
|
||||
self._input_hash = current_hash
|
||||
@@ -56,3 +56,7 @@ class CachingStep(AbstractStep[T], Generic[T]):
|
||||
|
||||
def reset(self) -> None:
|
||||
self._input_hash = None
|
||||
|
||||
def __getattr__(self, name: str) -> Any:
|
||||
"""Delegate unknown attributes to the wrapped step."""
|
||||
return getattr(self._step, name)
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
import json
|
||||
from dataclasses import asdict
|
||||
import os
|
||||
from typing import Any
|
||||
|
||||
|
||||
class DataclassJSONEncoder(json.JSONEncoder):
|
||||
def default(self, o: Any) -> Any:
|
||||
if hasattr(o, '__dataclass_fields__'):
|
||||
return asdict(o)
|
||||
if isinstance(o, BaseException):
|
||||
return str(o)
|
||||
try:
|
||||
return super().default(o)
|
||||
except TypeError:
|
||||
return str(o)
|
||||
|
||||
|
||||
def custom_json_dumps(obj: Any) -> str:
|
||||
return json.dumps(obj, cls=DataclassJSONEncoder)
|
||||
|
||||
def sigchld_handler(signum: int, frame: Any) -> None:
|
||||
"""Reap zombie processes."""
|
||||
while True:
|
||||
try:
|
||||
pid, status = os.waitpid(-1, os.WNOHANG)
|
||||
if pid == 0:
|
||||
break
|
||||
except ChildProcessError:
|
||||
break
|
||||
|
||||
+231
@@ -0,0 +1,231 @@
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import traceback
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import Any, AsyncIterator, Dict, List, Optional, TypeVar
|
||||
|
||||
from fastapi import FastAPI, Response, WebSocket, WebSocketDisconnect
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import FileResponse
|
||||
|
||||
from app import App
|
||||
from enironment import SharedStateConflictError, get_step
|
||||
from utils import custom_json_dumps
|
||||
from processing import reset_caches
|
||||
from secondary import SecondaryManager
|
||||
from steps.step import CachingStep
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
FRONTEND_DIR = os.path.join(os.path.dirname(__file__), '../frontend')
|
||||
|
||||
T = TypeVar('T')
|
||||
|
||||
|
||||
def _merge_dicts(a: Dict[str, T], b: Dict[str, T]) -> Dict[str, T]:
|
||||
result: Dict[str, T] = {}
|
||||
for k in (a.keys() | b.keys()):
|
||||
if k not in a or k not in b:
|
||||
result[k] = a.get(k, b.get(k)) # type: ignore[assignment]
|
||||
continue
|
||||
if (
|
||||
isinstance(a[k], dict)
|
||||
and isinstance(b[k], dict)
|
||||
):
|
||||
result[k] = _merge_dicts(a[k], b[k]) # type: ignore[arg-type,assignment]
|
||||
else:
|
||||
result[k] = b[k]
|
||||
return result
|
||||
|
||||
|
||||
class WebApp:
|
||||
|
||||
def __init__(self, core: App, port: int) -> None:
|
||||
self.core = core
|
||||
self.port = port
|
||||
|
||||
self.ws_connections: Dict[WebSocket, Dict[str, Any]] = {}
|
||||
self._event_loop: Optional[asyncio.AbstractEventLoop] = None
|
||||
self.secondaryManager: Optional[SecondaryManager] = None
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(fastapi_app: FastAPI) -> AsyncIterator[None]:
|
||||
self._event_loop = asyncio.get_event_loop()
|
||||
self.secondaryManager = SecondaryManager(
|
||||
urls=os.getenv("SECONDARY_BRENCHER", ""),
|
||||
on_update=lambda: self.broadcast_all(),
|
||||
)
|
||||
yield
|
||||
|
||||
self.app = FastAPI(lifespan=lifespan)
|
||||
self.app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=["*"],
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
self.app.get("/")(self.serve_index)
|
||||
self.app.get("/state")(self.serve_state)
|
||||
self.app.get("/branches")(self.serve_branches_route)
|
||||
self.app.get("/{path:path}")(self.serve_static)
|
||||
self.app.websocket("/ws")(self.websocket_endpoint)
|
||||
|
||||
# --- State assembly (local + secondary) ---
|
||||
|
||||
def get_global_envs_to_emit(self) -> Any:
|
||||
local_envs = self.core.get_local_envs_to_emit()
|
||||
merged: Dict[str, Any] = local_envs
|
||||
for connector in self.secondaryManager or []:
|
||||
v = connector.environments
|
||||
common_keys = set(merged.keys()) & set(v.keys())
|
||||
if len(common_keys) > 0:
|
||||
self._schedule_async(self.broadcast_error({'message': f"Conflict: both master and secondary have environment with id {common_keys}"}))
|
||||
merged = _merge_dicts(merged, v)
|
||||
return merged
|
||||
|
||||
def get_global_branches_to_emit(self) -> Dict[str, Dict[str, List[Any]]]:
|
||||
local_branches: Dict[str, Dict[str, List[Any]]] = self.core.get_local_branches_to_emit()
|
||||
merged: Dict[str, Dict[str, List[Any]]] = local_branches
|
||||
for connector in self.secondaryManager or []:
|
||||
merged = _merge_dicts(merged, connector.branches)
|
||||
return merged
|
||||
|
||||
# --- Async scheduling from sync threads ---
|
||||
|
||||
def _schedule_async(self, coro: Any) -> None:
|
||||
if self._event_loop is not None and self._event_loop.is_running():
|
||||
asyncio.run_coroutine_threadsafe(coro, self._event_loop)
|
||||
|
||||
# --- Broadcasting ---
|
||||
|
||||
async def broadcast(self, event: str, data: Any) -> None:
|
||||
"""Broadcast a message to all connected WebSocket clients that have not seen this data yet."""
|
||||
disconnected = set()
|
||||
message = custom_json_dumps({event: data})
|
||||
|
||||
for websocket in self.ws_connections.keys():
|
||||
if self.ws_connections[websocket].get(event) == message:
|
||||
continue
|
||||
try:
|
||||
await websocket.send_text(message)
|
||||
self.ws_connections[websocket][event] = message
|
||||
except Exception as e:
|
||||
logger.error(f"Error sending to websocket: {e}")
|
||||
disconnected.add(websocket)
|
||||
|
||||
for ws in disconnected:
|
||||
self.ws_connections.pop(ws, None)
|
||||
|
||||
async def broadcast_all(self) -> None:
|
||||
await self.broadcast("branches", self.get_global_branches_to_emit())
|
||||
await self.broadcast("environments", self.get_global_envs_to_emit())
|
||||
|
||||
async def broadcast_branches(self, data: Any) -> None:
|
||||
await self.broadcast("branches", data)
|
||||
|
||||
async def broadcast_environments(self, data: Any) -> None:
|
||||
await self.broadcast("environments", data)
|
||||
|
||||
async def broadcast_error(self, data: Any) -> None:
|
||||
await self.broadcast("error", data)
|
||||
|
||||
def emit_envs(self) -> None:
|
||||
"""Sync callback used by App's processing thread to push updates to clients."""
|
||||
self._schedule_async(self.broadcast_branches(self.get_global_branches_to_emit()))
|
||||
self._schedule_async(self.broadcast_environments(self.get_global_envs_to_emit()))
|
||||
|
||||
# --- Routes ---
|
||||
|
||||
async def serve_index(self) -> FileResponse:
|
||||
return FileResponse(os.path.join(FRONTEND_DIR, 'index.html'))
|
||||
|
||||
async def serve_state(self) -> Response:
|
||||
return Response(content=custom_json_dumps(self.get_global_envs_to_emit()), media_type="application/json")
|
||||
|
||||
async def serve_branches_route(self) -> Response:
|
||||
return Response(content=custom_json_dumps(self.get_global_branches_to_emit()), media_type="application/json")
|
||||
|
||||
async def serve_static(self, path: str) -> FileResponse:
|
||||
file_path = os.path.join(FRONTEND_DIR, path)
|
||||
if os.path.exists(file_path) and os.path.isfile(file_path):
|
||||
return FileResponse(file_path)
|
||||
return FileResponse(os.path.join(FRONTEND_DIR, 'index.html'))
|
||||
|
||||
async def websocket_endpoint(self, websocket: WebSocket) -> None:
|
||||
await websocket.accept()
|
||||
self.ws_connections[websocket] = {}
|
||||
|
||||
try:
|
||||
# Send initial state on connect and record it so duplicate broadcasts are suppressed
|
||||
branches_payload = custom_json_dumps({"branches": self.get_global_branches_to_emit()})
|
||||
envs_payload = custom_json_dumps({"environments": self.get_global_envs_to_emit()})
|
||||
await websocket.send_text(branches_payload)
|
||||
self.ws_connections[websocket]["branches"] = branches_payload
|
||||
await websocket.send_text(envs_payload)
|
||||
self.ws_connections[websocket]["environments"] = envs_payload
|
||||
|
||||
while True:
|
||||
data = await websocket.receive_text()
|
||||
message = json.loads(data)
|
||||
|
||||
if "update" in message:
|
||||
update_data = message.get("update") or {}
|
||||
logger.info(f"Received environment update: {update_data}")
|
||||
if self.secondaryManager:
|
||||
await self.secondaryManager.send({"update": update_data})
|
||||
id = update_data.get('id', '')
|
||||
if id == '':
|
||||
reset_caches(list(self.core.environments.values()))
|
||||
elif id not in self.core.environments.keys() and id not in {j for it in self.secondaryManager or [] for j in it.environments.keys()}:
|
||||
logger.warning(f"Received update for unknown environment id {id}")
|
||||
continue
|
||||
elif id in self.core.environments.keys():
|
||||
env = self.core.environments.get(id, None)
|
||||
expected_token = update_data.get('token', '')
|
||||
try:
|
||||
if 'branches' in update_data:
|
||||
if not env:
|
||||
raise RuntimeError(f"Unknown env {update_data.get('id', '')}")
|
||||
env.state.set_branches(update_data.get('branches', []), expected_token=expected_token)
|
||||
if 'dry' in update_data:
|
||||
if not env:
|
||||
raise RuntimeError(f"Unknown env {update_data.get('id', '')}")
|
||||
env.state.set_dry(bool(update_data['dry']), expected_token)
|
||||
except SharedStateConflictError as conflict:
|
||||
# if env:
|
||||
# current_state = env.state.progress()
|
||||
# await self.broadcast_error({
|
||||
# 'code': 'BRANCH_STATE_CONFLICT',
|
||||
# 'envId': id,
|
||||
# 'message': str(conflict),
|
||||
# 'current_state': {
|
||||
# 'branches': current_state.branches,
|
||||
# 'token': current_state.token,
|
||||
# },
|
||||
# })
|
||||
continue
|
||||
|
||||
if env:
|
||||
p = get_step(env.pipeline, type(env.state))
|
||||
if isinstance(p, CachingStep):
|
||||
p.reset()
|
||||
logger.info(f"Updated environment {env.id} branches to {update_data.get('branches')}, dry={update_data.get('dry')}")
|
||||
self.core.environment_update_event.set()
|
||||
|
||||
await self.broadcast_environments(self.get_global_envs_to_emit())
|
||||
|
||||
except WebSocketDisconnect:
|
||||
self.ws_connections.pop(websocket, None)
|
||||
except Exception as e:
|
||||
logger.error(f"WebSocket error: {e}:{traceback.format_exc()}")
|
||||
await self.broadcast_error({'message': f'{e}:{traceback.format_exc()}'})
|
||||
|
||||
# --- Run ---
|
||||
|
||||
def start(self) -> None:
|
||||
import uvicorn
|
||||
uvicorn.run(self.app, host='0.0.0.0', port=self.port, log_level='info')
|
||||
+7
-2
@@ -17,12 +17,17 @@ services:
|
||||
- VIRTUAL_HOST=brencher.rudolf.keenetic.link
|
||||
- LETSENCRYPT_HOST=brencher.rudolf.keenetic.link
|
||||
- PROFILES=brencher2,torrserv_proxy,immich,registry,gmail_mcp
|
||||
- SLAVE_BRENCHER=http://host.docker.internal:5002
|
||||
# - SLAVE_BRENCHER='http://100.70.193.97:5002' .venv/bin/python3 backend/app.py brencher_localX
|
||||
- SECONDARY_BRENCHER=http://host.docker.internal:5002
|
||||
deploy:
|
||||
replicas: 1
|
||||
restart_policy:
|
||||
condition: on-failure
|
||||
placement:
|
||||
max_replicas_per_node: 1
|
||||
constraints:
|
||||
- node.hostname==odroid
|
||||
# - node.hostname==rudolf-T
|
||||
mode: replicated
|
||||
networks:
|
||||
- default
|
||||
- brencher-net
|
||||
|
||||
+45
-11
@@ -30,6 +30,8 @@ let environmentsRaw = [];
|
||||
const selectedBranchesByEnv = {};
|
||||
// serverSelectedBranchesByEnv mirrors server's last known selection for diffing
|
||||
const serverSelectedBranchesByEnv = {};
|
||||
// serverTokenByEnv tracks current optimistic concurrency token per environment
|
||||
const serverTokenByEnv = {};
|
||||
// columnsByEnv: envId -> { columnName: { branchName: value } }
|
||||
// Populated from any step result that contains a `columns` key (e.g. GitUnmerge)
|
||||
const columnsByEnv = {};
|
||||
@@ -91,12 +93,14 @@ function getCommitDropdownOptions(envId, branchName, desiredCommit) {
|
||||
if (commits.length) {
|
||||
const head = commits[0];
|
||||
const headShort = head.hexsha ? head.hexsha.substring(0, 8) : '';
|
||||
options += `<option value="HEAD" ${desiredCommit === 'HEAD' ? 'selected' : ''}>HEAD (${headShort} - ${(head.author || '').substring(0, 25)} - ${head.message ? head.message.substring(0, 50) + (head.message.length > 50 ? '...' : '') : ''})</option>`;
|
||||
const headMsg = escapeHtml(head.message || '');
|
||||
options += `<option value="HEAD" ${desiredCommit === 'HEAD' ? 'selected' : ''}>HEAD (${headShort} - ${(head.author || '').substring(0, 25)} - ${headMsg ? headMsg.substring(0, 50) + (headMsg.length > 50 ? '...' : '') : ''})</option>`;
|
||||
commits.forEach(c => {
|
||||
const full = c.hexsha;
|
||||
const shortHash = full ? full.substring(0, 8) : '';
|
||||
const isSelected = desiredCommit === full || desiredCommit === shortHash;
|
||||
options += `<option value="${full}" ${isSelected ? 'selected' : ''}>${shortHash} - ${(c.author || '').substring(0, 25)} - ${c.message ? c.message.substring(0, 50) + (c.message.length > 50 ? '...' : '') : ''}</option>`;
|
||||
const cMsg = escapeHtml(c.message || '');
|
||||
options += `<option value="${full}" ${isSelected ? 'selected' : ''}>${shortHash} - ${(c.author || '').substring(0, 25)} - ${cMsg ? cMsg.substring(0, 50) + (cMsg.length > 50 ? '...' : '') : ''}</option>`;
|
||||
});
|
||||
} else {
|
||||
options += `<option value="HEAD" ${desiredCommit === 'HEAD' ? 'selected' : ''}>HEAD (no commits)</option>`;
|
||||
@@ -133,9 +137,20 @@ function escapeHtml(str) {
|
||||
}
|
||||
|
||||
let renderedBranches = []
|
||||
let renderedDryRunState = {}
|
||||
let renderedColumnsSignature = ''
|
||||
|
||||
function getColumnsSignature() {
|
||||
return JSON.stringify(columnsByEnv);
|
||||
}
|
||||
|
||||
function renderBranches() {
|
||||
if (JSON.stringify(filteredBranches) === JSON.stringify(renderedBranches) && JSON.stringify(dryRunByEnv) === JSON.stringify(renderedDryRunState)) {
|
||||
const columnsSignature = getColumnsSignature();
|
||||
if (
|
||||
JSON.stringify(filteredBranches) === JSON.stringify(renderedBranches) &&
|
||||
JSON.stringify(dryRunByEnv) === JSON.stringify(renderedDryRunState) &&
|
||||
columnsSignature === renderedColumnsSignature
|
||||
) {
|
||||
return
|
||||
}
|
||||
checkForPendingChanges();
|
||||
@@ -161,7 +176,7 @@ function renderBranches() {
|
||||
<h3 class="env-title">
|
||||
Environment: ${envName || envId}
|
||||
<button class="dry-run-btn ${dryRunByEnv[envId] ? 'dry-run-active' : ''}" data-env="${envId}" title="${dryRunByEnv[envId] ? 'Dry run on — click to resume' : 'Running — click to pause (dry run)'}">
|
||||
${dryRunByEnv[envId] ? '⏸' : '▶'}
|
||||
${dryRunByEnv[envId] ? '▶' : '⏸'}
|
||||
</button>
|
||||
</h3>
|
||||
<table class="branches-table">
|
||||
@@ -242,15 +257,15 @@ function renderBranches() {
|
||||
branchesList.querySelectorAll('.dry-run-btn').forEach(btn => {
|
||||
btn.onclick = e => {
|
||||
const envId = e.currentTarget.dataset.env;
|
||||
dryRunByEnv[envId] = !dryRunByEnv[envId];
|
||||
if (ws && ws.readyState === WebSocket.OPEN) {
|
||||
ws.send(JSON.stringify({ update: { id: envId, dry: dryRunByEnv[envId] } }));
|
||||
ws.send(JSON.stringify({ update: { id: envId, dry: !dryRunByEnv[envId], token: serverTokenByEnv[envId] } }));
|
||||
}
|
||||
filterBranches();
|
||||
};
|
||||
});
|
||||
renderedBranches = filteredBranches;
|
||||
renderedBranches = [...filteredBranches];
|
||||
renderedDryRunState = {...dryRunByEnv};
|
||||
renderedColumnsSignature = columnsSignature;
|
||||
}
|
||||
|
||||
function updateBranchCommit(envId, branch, commit) {
|
||||
@@ -415,8 +430,11 @@ function setupWebSockets() {
|
||||
environmentsRaw.forEach((envObj) => {
|
||||
if (!envObj) return;
|
||||
const envId = envObj.id || envObj.name || 'unknown';
|
||||
// Always sync dry run state from server
|
||||
// Server state is the source of truth for dry-run rendering.
|
||||
dryRunByEnv[envId] = !!envObj.dry;
|
||||
if (typeof envObj.branches_token === 'string') {
|
||||
serverTokenByEnv[envId] = envObj.branches_token;
|
||||
}
|
||||
if (!isChangesPending()) {
|
||||
if (Array.isArray(envObj.branches) && envObj.branches.length > 0) {
|
||||
if (Array.isArray(envObj.branches[0])) {
|
||||
@@ -461,6 +479,22 @@ function setupWebSockets() {
|
||||
showStatus('Environments updated.');
|
||||
} else if ('error' in message) {
|
||||
console.error('Error from server:', message.error);
|
||||
if (message.error && message.error.code === 'BRANCH_STATE_CONFLICT' && message.error.envId) {
|
||||
const envId = message.error.envId;
|
||||
const current = message.error.current_state || {};
|
||||
const hadLocalSelection = Object.prototype.hasOwnProperty.call(selectedBranchesByEnv, envId);
|
||||
if (Array.isArray(current.branches)) {
|
||||
serverSelectedBranchesByEnv[envId] = [...current.branches];
|
||||
// Preserve the user's pending edits if they already changed selection locally.
|
||||
if (!hadLocalSelection) {
|
||||
selectedBranchesByEnv[envId] = [...current.branches];
|
||||
}
|
||||
}
|
||||
if (typeof current.token === 'string') {
|
||||
serverTokenByEnv[envId] = current.token;
|
||||
}
|
||||
filterBranches();
|
||||
}
|
||||
showStatus(message.error.message || 'Unknown error', true);
|
||||
}
|
||||
checkForPendingChanges();
|
||||
@@ -484,10 +518,10 @@ function updateEnvironment() {
|
||||
const serverSel = [...(serverSelectedBranchesByEnv[envId] || [])].sort();
|
||||
if (JSON.stringify(localSel) !== JSON.stringify(serverSel)) {
|
||||
if (ws && ws.readyState === WebSocket.OPEN) {
|
||||
ws.send(JSON.stringify({ update: { id: envId, branches: selectedBranchesByEnv[envId] } }));
|
||||
ws.send(JSON.stringify({ update: { id: envId, branches: selectedBranchesByEnv[envId], token: serverTokenByEnv[envId] } }));
|
||||
// Keep pending-state tracking aligned with what we just submitted.
|
||||
serverSelectedBranchesByEnv[envId] = [...(selectedBranchesByEnv[envId] || [])];
|
||||
}
|
||||
// Optimistically sync server state
|
||||
serverSelectedBranchesByEnv[envId] = [...selectedBranchesByEnv[envId]];
|
||||
branchFilter.value = '';
|
||||
}
|
||||
});
|
||||
|
||||
+1
-1
@@ -210,7 +210,7 @@ input[type="checkbox"] {
|
||||
line-height: 1;
|
||||
padding: 2px 6px;
|
||||
border-radius: 4px;
|
||||
color: #28a745;
|
||||
color: #aaaaaa;
|
||||
transition: color 0.15s, background 0.15s;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
-r requirements.txt
|
||||
|
||||
pytest
|
||||
pytest-asyncio
|
||||
playwright
|
||||
pytest-playwright
|
||||
mypy
|
||||
flake8
|
||||
ty
|
||||
types-PyYAML
|
||||
types-requests
|
||||
types-docker
|
||||
testcontainers
|
||||
@@ -5,12 +5,3 @@ gitpython
|
||||
python-dotenv
|
||||
docker
|
||||
PyYAML
|
||||
|
||||
pytest
|
||||
pytest-asyncio
|
||||
mypy
|
||||
ty
|
||||
types-PyYAML
|
||||
types-requests
|
||||
types-docker
|
||||
testcontainers
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
from enironment import Environment
|
||||
from steps.docker_plain import DockerImageBuild, DockerContainerCheck, DockerContainerDeploy
|
||||
from steps.git import GitClone, CheckoutMerged, GitUnmerge
|
||||
from steps.shared_state import SharedStateHolderInMemory
|
||||
|
||||
clone = GitClone(url="https://github.com/rudolf1/brencher.git")
|
||||
|
||||
container_check = DockerContainerCheck(
|
||||
container_name="test_plain-container"
|
||||
)
|
||||
|
||||
unmerge = GitUnmerge(wd=clone, check=container_check)
|
||||
|
||||
state = SharedStateHolderInMemory(unmerge=unmerge)
|
||||
|
||||
checkoutMerged = CheckoutMerged(clone,
|
||||
desired_branches=state,
|
||||
push=False,
|
||||
git_user_email="rudolfss13@gmail.com",
|
||||
git_user_name="test_bot"
|
||||
)
|
||||
|
||||
# Step 1: Build the image
|
||||
image_build = DockerImageBuild(
|
||||
wd=checkoutMerged,
|
||||
dockerfile_path="Dockerfile_test_app",
|
||||
image_name="test_plain",
|
||||
image_tag=lambda: "auto-" + checkoutMerged.progress().version,
|
||||
nocache=False,
|
||||
)
|
||||
|
||||
# Step 3: Deploy the container
|
||||
container_deploy = DockerContainerDeploy(
|
||||
image_build=image_build,
|
||||
container_name="test_plain-container",
|
||||
ports={"5001/tcp": 5002},
|
||||
environment={"PROFILES": "no_profiles"},
|
||||
restart_policy={"Name": "unless-stopped"},
|
||||
)
|
||||
|
||||
__all__ = ["test_local1"]
|
||||
test_local1 = Environment(
|
||||
id="test_local1",
|
||||
state=state,
|
||||
pipeline=[
|
||||
clone,
|
||||
state,
|
||||
checkoutMerged,
|
||||
image_build,
|
||||
container_deploy,
|
||||
container_deploy,
|
||||
unmerge
|
||||
]
|
||||
)
|
||||
+25
-11
@@ -4,9 +4,10 @@ Pytest configuration and shared fixtures for integration tests.
|
||||
This file configures the Python path to allow importing backend modules
|
||||
and provides shared fixtures for all tests.
|
||||
"""
|
||||
from typing import Generator, Callable, Protocol
|
||||
from typing import Any, Generator, Callable, Protocol, TypeVar
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from tests.test_remote_repo import RemoteRepoHelper
|
||||
|
||||
@@ -26,25 +27,38 @@ import pytest
|
||||
class EventuallyFn(Protocol):
|
||||
def __call__(
|
||||
self,
|
||||
assert_fn: Callable[[], bool],
|
||||
timeout: float = 2.0,
|
||||
interval: float = 0.05,
|
||||
assert_fn: Callable[[], None],
|
||||
timeout: float = 5.0,
|
||||
interval: float = 0.5,
|
||||
) -> None: ...
|
||||
|
||||
T = TypeVar('T')
|
||||
|
||||
def assert_equal(a: T, b:T, message: str) -> None:
|
||||
if a != b:
|
||||
raise AssertionError(f"{message}: {a} != {b}")
|
||||
|
||||
def check_state(url: str, predicate: Callable[[Any], None]) -> None:
|
||||
response = requests.get(f"{url}/state", timeout=5)
|
||||
assert response.status_code == 200, f"Expected status code 200, got {response.status_code}"
|
||||
state_data = response.json()
|
||||
try:
|
||||
predicate(state_data)
|
||||
except Exception as e:
|
||||
raise AssertionError(f"Predicate check failed with error: {e}\nState data: {state_data}") from e
|
||||
|
||||
@pytest.fixture
|
||||
def eventually() -> EventuallyFn:
|
||||
def _eventually(assert_fn: Callable[[], bool], timeout: float = 2.0, interval: float = 0.05) -> None:
|
||||
def _eventually(assert_fn: Callable[[], None], timeout: float = 5.0, interval: float = 0.5) -> None:
|
||||
deadline = time.monotonic() + timeout
|
||||
last_error = None
|
||||
last_error: BaseException | None = None
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
res = assert_fn()
|
||||
if res:
|
||||
return
|
||||
except AssertionError as err:
|
||||
assert_fn()
|
||||
return
|
||||
except Exception as err:
|
||||
last_error = err
|
||||
time.sleep(interval)
|
||||
time.sleep(interval)
|
||||
|
||||
raise AssertionError(f"Condition not met within {timeout:.2f}s") from last_error
|
||||
|
||||
|
||||
@@ -0,0 +1,211 @@
|
||||
import logging
|
||||
from contextlib import contextmanager
|
||||
from typing import Generator
|
||||
|
||||
from playwright.sync_api import Locator, Page, sync_playwright
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BrencherPage:
|
||||
|
||||
def __init__(self, page: Page, env_id: str, action_logger: logging.Logger) -> None:
|
||||
self._page = page
|
||||
self._env_id = env_id
|
||||
self._logger = action_logger
|
||||
|
||||
def _dry_run_button(self) -> Locator:
|
||||
button = self._page.locator(f'h3:has-text("{self._env_id}") button.dry-run-btn')
|
||||
button.wait_for(state="visible", timeout=10000)
|
||||
return button
|
||||
|
||||
def _apply_changes_button(self) -> Locator:
|
||||
button = self._page.locator("#apply-changes")
|
||||
button.wait_for(state="visible", timeout=10000)
|
||||
return button
|
||||
|
||||
def _branch_checkbox(self, branch_name: str) -> Locator:
|
||||
checkbox = self._page.locator(
|
||||
f'input[type="checkbox"][data-env="{self._env_id}"][value="{branch_name}"]'
|
||||
)
|
||||
checkbox.wait_for(state="visible", timeout=10000)
|
||||
return checkbox
|
||||
|
||||
def _desired_commit_select(self, branch_name: str) -> Locator:
|
||||
select = self._page.locator(
|
||||
f'select.desired-commit[data-env="{self._env_id}"][data-branch="{branch_name}"]'
|
||||
)
|
||||
select.wait_for(state="visible", timeout=10000)
|
||||
return select
|
||||
|
||||
def verify_dry_run_on(self) -> None:
|
||||
btn_title = self._dry_run_button().get_attribute("title") or ""
|
||||
assert "Dry run on" in btn_title, f"Expected dry run to be active, got title: {btn_title!r}"
|
||||
self._logger.info(f"Dry-run button title before click: {btn_title!r}")
|
||||
|
||||
def set_dry_run_off(self) -> None:
|
||||
self._dry_run_button().click()
|
||||
self._logger.info("Clicked dry-run button to disable dry run")
|
||||
|
||||
def set_branch_unselected(self, branch_name: str) -> None:
|
||||
checkbox = self._branch_checkbox(branch_name)
|
||||
if checkbox.is_checked():
|
||||
checkbox.uncheck()
|
||||
self._logger.info("Unchecked branch '%s'", branch_name)
|
||||
|
||||
def set_branch_desired_commit(self, branch_name: str, commit_hash: str) -> None:
|
||||
self._desired_commit_select(branch_name).select_option(value=commit_hash)
|
||||
self._logger.info("Set desired commit for branch '%s' to '%s'", branch_name, commit_hash)
|
||||
|
||||
def set_apply_changes(self) -> None:
|
||||
self._apply_changes_button().click()
|
||||
self._logger.info("Clicked Apply Changes")
|
||||
|
||||
def set_simulate_branch_state_conflict(self) -> None:
|
||||
self._page.evaluate(
|
||||
"""({ envId }) => {
|
||||
const currentToken =
|
||||
(typeof serverTokenByEnv !== 'undefined' && serverTokenByEnv[envId])
|
||||
? serverTokenByEnv[envId]
|
||||
: 'server-token';
|
||||
const staleToken = `${currentToken}-stale`;
|
||||
if (typeof serverTokenByEnv !== 'undefined') {
|
||||
// Simulate stale client-side token so next apply uses an outdated value.
|
||||
serverTokenByEnv[envId] = staleToken;
|
||||
}
|
||||
}""",
|
||||
{
|
||||
"envId": self._env_id,
|
||||
},
|
||||
)
|
||||
self._logger.info("Set stale branch token for '%s'", self._env_id)
|
||||
|
||||
def click_refresh(self) -> None:
|
||||
self._page.locator("#refresh-branches").click()
|
||||
self._logger.info("Clicked refresh button")
|
||||
|
||||
def verify_dry_run_off(self) -> None:
|
||||
self._page.wait_for_function(
|
||||
"""(targetEnvId) => {
|
||||
const h3s = [...document.querySelectorAll('h3.env-title')];
|
||||
const h3 = h3s.find(el => el.textContent.includes(targetEnvId));
|
||||
if (!h3) return false;
|
||||
const btn = h3.querySelector('button.dry-run-btn');
|
||||
return btn && !btn.title.includes('Dry run on');
|
||||
}""",
|
||||
arg=self._env_id,
|
||||
timeout=10000,
|
||||
)
|
||||
|
||||
def verify_apply_changes_visible(self) -> None:
|
||||
self._page.wait_for_function(
|
||||
"""() => {
|
||||
const btn = document.querySelector('#apply-changes');
|
||||
return !!btn && !btn.classList.contains('hidden');
|
||||
}""",
|
||||
timeout=10000,
|
||||
)
|
||||
self._logger.info("Apply Changes button is visible")
|
||||
|
||||
def verify_apply_changes_hidden(self) -> None:
|
||||
self._page.wait_for_function(
|
||||
"""() => {
|
||||
const btn = document.querySelector('#apply-changes');
|
||||
return !!btn && btn.classList.contains('hidden');
|
||||
}""",
|
||||
timeout=10000,
|
||||
)
|
||||
self._logger.info("Apply Changes button is hidden")
|
||||
|
||||
def verify_branch_column_value(self, branch_name: str, column_name: str, expected_value: str, timeout_ms: int = 10000) -> None:
|
||||
self._page.wait_for_function(
|
||||
"""({ envId, branchName, columnName, expectedValue }) => {
|
||||
const row = document.querySelector(`tr.branch-row[data-env="${envId}"][data-branch="${branchName}"]`);
|
||||
if (!row) return false;
|
||||
const table = row.closest('table.branches-table');
|
||||
if (!table) return false;
|
||||
const headers = [...table.querySelectorAll('thead th')].map(th => (th.textContent || '').trim());
|
||||
const colIdx = headers.indexOf(columnName);
|
||||
if (colIdx === -1) return false;
|
||||
const cells = [...row.querySelectorAll('td')];
|
||||
if (cells.length <= colIdx) return false;
|
||||
return ((cells[colIdx].textContent || '').trim() === expectedValue);
|
||||
}""",
|
||||
arg={
|
||||
"envId": self._env_id,
|
||||
"branchName": branch_name,
|
||||
"columnName": column_name,
|
||||
"expectedValue": expected_value,
|
||||
},
|
||||
timeout=timeout_ms,
|
||||
)
|
||||
self._logger.info(
|
||||
"Verified column '%s' for branch '%s' equals '%s'",
|
||||
column_name,
|
||||
branch_name,
|
||||
expected_value,
|
||||
)
|
||||
|
||||
def verify_pipeline_step_status(self, step_name: str, expected_status: str, timeout_ms: int = 10000) -> None:
|
||||
"""
|
||||
Verify that a pipeline step reaches the expected status by polling the page DOM.
|
||||
|
||||
Args:
|
||||
step_name: Name of the pipeline step (e.g., "DockerContainerDeploy")
|
||||
expected_status: Expected status value ("running", "ok", "error", or "dry-run")
|
||||
timeout_ms: Timeout in milliseconds (default 10000)
|
||||
"""
|
||||
# JavaScript to check pipeline step status
|
||||
js_check = """(stepName, expectedStatus) => {
|
||||
const jobItems = document.querySelectorAll('.job-item');
|
||||
for (const item of jobItems) {
|
||||
const headerText = item.querySelector('.job-header')?.textContent || '';
|
||||
if (headerText.includes(stepName)) {
|
||||
const spinner = item.querySelector('.step-spinner');
|
||||
const errorIndicator = item.querySelector('span[title="Error"]');
|
||||
|
||||
if (expectedStatus === 'running') {
|
||||
return spinner !== null;
|
||||
} else if (expectedStatus === 'ok') {
|
||||
return spinner === null && errorIndicator === null;
|
||||
} else if (expectedStatus === 'error') {
|
||||
return errorIndicator !== null;
|
||||
} else if (expectedStatus === 'dry-run') {
|
||||
// For dry-run, check the spoiler content
|
||||
const spoiler = item.querySelector('.job-spoiler');
|
||||
if (spoiler && spoiler.style.display !== 'none') {
|
||||
const statusText = spoiler.textContent;
|
||||
return statusText.includes('dry-run') || statusText.includes('dry_run');
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}"""
|
||||
|
||||
self._page.wait_for_function(
|
||||
js_check,
|
||||
arg=[step_name, expected_status],
|
||||
timeout=timeout_ms,
|
||||
)
|
||||
self._logger.info(f"Pipeline step '{step_name}' reached status '{expected_status}'")
|
||||
|
||||
|
||||
@contextmanager
|
||||
def brencher_page(
|
||||
app_link: str,
|
||||
env_id: str,
|
||||
action_logger: logging.Logger | None = None,
|
||||
) -> Generator[BrencherPage, None, None]:
|
||||
active_logger = action_logger or logger
|
||||
with sync_playwright() as playwright:
|
||||
browser = playwright.chromium.launch(headless=True)
|
||||
page = browser.new_page()
|
||||
page.goto(app_link)
|
||||
page.wait_for_load_state("networkidle")
|
||||
try:
|
||||
yield BrencherPage(page, env_id, active_logger)
|
||||
finally:
|
||||
browser.close()
|
||||
@@ -0,0 +1,127 @@
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from typing import Any, Generator
|
||||
|
||||
import docker
|
||||
import pytest
|
||||
import requests
|
||||
import socket
|
||||
|
||||
from app import App
|
||||
from tests.configs import nginx
|
||||
from tests.conftest import EventuallyFn, assert_equal
|
||||
from tests.playwright_helper import brencher_page
|
||||
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
APP_PORT = 5003
|
||||
APP_URL = f"http://localhost:{APP_PORT}"
|
||||
CONTAINER_NAME = "test_plain-container"
|
||||
|
||||
# Module-level reference to server thread to ensure proper cleanup
|
||||
_server_thread: threading.Thread | None = None
|
||||
|
||||
|
||||
def _is_port_free(port: int, timeout: float = 5.0) -> bool:
|
||||
"""Check if a port is available for binding."""
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
result = sock.connect_ex(('127.0.0.1', port))
|
||||
sock.close()
|
||||
if result != 0:
|
||||
return True
|
||||
except Exception:
|
||||
return True
|
||||
time.sleep(0.1)
|
||||
return False
|
||||
|
||||
|
||||
class TestApplyChangesPlaywright:
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def setup_teardown(self) -> Generator[None, Any, None]:
|
||||
global _server_thread
|
||||
|
||||
def stop_and_remove_container() -> None:
|
||||
client = docker.from_env()
|
||||
try:
|
||||
container = client.containers.get(CONTAINER_NAME)
|
||||
container.stop(timeout=10)
|
||||
container.remove(force=True)
|
||||
logger.info("Container %s stopped and removed", CONTAINER_NAME)
|
||||
except Exception:
|
||||
logger.info("Container %s not found, no need to stop/remove", CONTAINER_NAME)
|
||||
|
||||
# Ensure port is free before starting
|
||||
_is_port_free(APP_PORT, timeout=5.0)
|
||||
|
||||
stop_and_remove_container()
|
||||
yield None
|
||||
|
||||
# Clean up server thread
|
||||
if _server_thread and _server_thread.is_alive():
|
||||
_server_thread.join(timeout=2.0)
|
||||
logger.info("Waited for server thread to finish")
|
||||
|
||||
# Ensure port is free after test
|
||||
_is_port_free(APP_PORT, timeout=5.0)
|
||||
|
||||
stop_and_remove_container()
|
||||
_server_thread = None
|
||||
|
||||
def test_apply_button_hides_after_apply_branch_change(self, eventually: EventuallyFn) -> None:
|
||||
global _server_thread
|
||||
|
||||
# Start app with one pre-selected branch so unchecking creates pending changes.
|
||||
env = nginx.test_local1
|
||||
env.state.set_branches([("test/main", "HEAD")])
|
||||
env.state.set_dry(True)
|
||||
app = App({env.id: env})
|
||||
_server_thread = threading.Thread(target=lambda: app.runWeb(APP_PORT), daemon=True)
|
||||
_server_thread.start()
|
||||
|
||||
eventually(
|
||||
lambda: assert_equal(
|
||||
requests.get(f"{APP_URL}/state", timeout=5).status_code,
|
||||
200,
|
||||
"Server did not respond with 200 OK within timeout",
|
||||
)
|
||||
)
|
||||
|
||||
with brencher_page(APP_URL, env.id) as page:
|
||||
page.set_branch_unselected("test/main")
|
||||
page.verify_apply_changes_visible()
|
||||
page.set_apply_changes()
|
||||
eventually(lambda: page.verify_apply_changes_hidden(), timeout=10, interval=0.5)
|
||||
|
||||
def test_apply_button_reappears_on_branch_conflict(self, eventually: EventuallyFn) -> None:
|
||||
global _server_thread
|
||||
|
||||
env = nginx.test_local1
|
||||
env.state.set_branches([("test/main", "HEAD")])
|
||||
env.state.set_dry(True)
|
||||
app = App({env.id: env})
|
||||
_server_thread = threading.Thread(target=lambda: app.runWeb(APP_PORT), daemon=True)
|
||||
_server_thread.start()
|
||||
|
||||
eventually(
|
||||
lambda: assert_equal(
|
||||
requests.get(f"{APP_URL}/state", timeout=5).status_code,
|
||||
200,
|
||||
"Server did not respond with 200 OK within timeout",
|
||||
)
|
||||
)
|
||||
|
||||
with brencher_page(APP_URL, env.id) as page:
|
||||
page.set_branch_unselected("test/main")
|
||||
page.verify_apply_changes_visible()
|
||||
# Set stale token in client JS object so backend returns a real conflict on apply.
|
||||
page.set_simulate_branch_state_conflict()
|
||||
page.set_apply_changes()
|
||||
eventually(lambda: page.verify_apply_changes_visible(), timeout=10, interval=0.5)
|
||||
@@ -1,5 +1,5 @@
|
||||
import pytest
|
||||
from enironment import wrap_in_cached
|
||||
from enironment import wrap_in_cached, SharedStateConflictError
|
||||
|
||||
from tests.test_remote_repo import RemoteRepoHelper
|
||||
|
||||
@@ -15,13 +15,26 @@ class TestCachedEnv:
|
||||
|
||||
repo_helper.env = wrap_in_cached(repo_helper.env)
|
||||
|
||||
repo_helper.env.branches = []
|
||||
repo_helper.set_desired_branches([])
|
||||
|
||||
with pytest.raises(BaseException, match="Empty branches set"):
|
||||
repo_helper.checkout_merged.progress()
|
||||
|
||||
repo_helper.env.branches = [('master', 'HEAD')]
|
||||
repo_helper.set_desired_branches([('master', 'HEAD')])
|
||||
result = repo_helper.checkout_merged.progress()
|
||||
|
||||
# Verify same auto branch is used with actual value checks
|
||||
assert result.commit_hash == commit1.hexsha, f"Expected {commit1.hexsha}, got {result.commit_hash}"
|
||||
|
||||
def test_inmemory_state_rejects_stale_token(self, repo_helper: RemoteRepoHelper) -> None:
|
||||
_ = repo_helper.create_commit(repo_helper.repo, "master", "master", "file1.txt", "content1", "Initial commit")
|
||||
_ = repo_helper.create_commit(repo_helper.repo, "master", "branch1", "file2.txt", "content2", "Branch1 commit")
|
||||
|
||||
initial_state = repo_helper.resolve_initial_branches.progress()
|
||||
initial_token = initial_state.token
|
||||
assert isinstance(initial_token, str)
|
||||
|
||||
repo_helper.resolve_initial_branches.set_branches([("master", "HEAD")], expected_token=initial_token)
|
||||
|
||||
with pytest.raises(SharedStateConflictError, match="State token mismatch"):
|
||||
repo_helper.resolve_initial_branches.set_branches([("branch1", "HEAD")], expected_token=initial_token)
|
||||
|
||||
@@ -6,6 +6,7 @@ from typing import Any
|
||||
import pytest
|
||||
|
||||
from enironment import AbstractStep, Environment
|
||||
from steps.shared_state import SharedStateHolderInMemory
|
||||
from steps.step import CachingStep, NotReadyException
|
||||
|
||||
|
||||
@@ -49,7 +50,7 @@ class FailingStep(AbstractStep[str]):
|
||||
|
||||
|
||||
def _make_env(steps: list[AbstractStep[Any]]) -> Environment:
|
||||
return Environment(id="test", branches=[], dry=True, repo="repo", pipeline=steps)
|
||||
return Environment(id="test", state=SharedStateHolderInMemory(unmerge=None), pipeline=steps)
|
||||
|
||||
|
||||
class TestCachingStep:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# import asyncio
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
@@ -8,7 +8,8 @@ import docker
|
||||
import pytest
|
||||
import requests
|
||||
from app import App
|
||||
from conftest import EventuallyFn # type: ignore[import-not-found]
|
||||
from configs import brencher_local1
|
||||
from .conftest import EventuallyFn, assert_equal
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -29,22 +30,24 @@ class TestDockerContainer:
|
||||
container.remove(force=True)
|
||||
logger.info("Container brencher_plain-container stopped and removed")
|
||||
|
||||
@pytest.mark.skip
|
||||
@pytest.mark.asyncio
|
||||
async def test_start(self, eventually: EventuallyFn) -> None: # type: ignore[no-any-unimported]
|
||||
async def test_start(self, eventually: EventuallyFn) -> None:
|
||||
logger.info(f"Starting")
|
||||
|
||||
app = App(cli_env_ids_str="brencher_local1")
|
||||
app = App({"brencher_local1": brencher_local1.brencher_local1})
|
||||
# app.runWeb(5001)
|
||||
processing = threading.Thread(target=lambda: app.runWeb(5001), daemon=True)
|
||||
processing.start()
|
||||
|
||||
# await asyncio.sleep(5000)
|
||||
|
||||
eventually(
|
||||
lambda: requests.get("http://localhost:5001/state", timeout=5000).status_code == 200,
|
||||
lambda: assert_equal(requests.get("http://localhost:5001/state", timeout=5000).status_code, 200, "Server did not respond with 200 OK within timeout"),
|
||||
20.0,
|
||||
1.0,
|
||||
)
|
||||
|
||||
# await asyncio.sleep(5000)
|
||||
|
||||
state_data = requests.get("http://localhost:5001/state", timeout=5000).json()
|
||||
logger.info(f"Application state: {json.dumps(state_data, indent=2)}")
|
||||
@@ -0,0 +1,182 @@
|
||||
import logging
|
||||
import socket
|
||||
import threading
|
||||
import time
|
||||
from typing import Any, Generator
|
||||
|
||||
import docker
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from app import App
|
||||
from tests.configs import nginx
|
||||
from tests.conftest import EventuallyFn, assert_equal
|
||||
from tests.playwright_helper import brencher_page
|
||||
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
APP_PORT = 5004
|
||||
APP_URL = f"http://localhost:{APP_PORT}"
|
||||
CONTAINER_NAME = "test_plain-container"
|
||||
TARGET_BRANCH = "test/main"
|
||||
|
||||
_server_thread: threading.Thread | None = None
|
||||
|
||||
|
||||
def _is_port_free(port: int, timeout: float = 5.0) -> bool:
|
||||
"""Check if a port is available for binding."""
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
result = sock.connect_ex(("127.0.0.1", port))
|
||||
sock.close()
|
||||
if result != 0:
|
||||
return True
|
||||
except Exception:
|
||||
return True
|
||||
time.sleep(0.1)
|
||||
return False
|
||||
|
||||
|
||||
def _get_state() -> dict[str, Any]:
|
||||
response = requests.get(f"{APP_URL}/state", timeout=5)
|
||||
assert response.status_code == 200, f"Expected status 200 from /state, got {response.status_code}"
|
||||
data = response.json()
|
||||
assert isinstance(data, dict), f"Expected dict from /state, got {type(data)}"
|
||||
return data
|
||||
|
||||
|
||||
def _get_deployed_short_hash(env_id: str, branch: str) -> str:
|
||||
state = _get_state()
|
||||
env_state = state.get(env_id)
|
||||
assert isinstance(env_state, dict), f"No environment state for {env_id}: {state}"
|
||||
|
||||
pipeline = env_state.get("pipeline")
|
||||
assert isinstance(pipeline, list), f"Pipeline is not list for {env_id}: {env_state}"
|
||||
|
||||
step = next((p for p in pipeline if p.get("name") == "GitUnmerge"), None)
|
||||
assert isinstance(step, dict), f"GitUnmerge step not found in pipeline: {pipeline}"
|
||||
assert not bool(step.get("is_running")), f"GitUnmerge is still running: {step}"
|
||||
|
||||
status = step.get("status")
|
||||
assert isinstance(status, dict), f"GitUnmerge status is not an object: {status}"
|
||||
|
||||
columns = status.get("columns")
|
||||
assert isinstance(columns, dict), f"GitUnmerge columns missing: {status}"
|
||||
|
||||
deployed = columns.get("Deployed")
|
||||
assert isinstance(deployed, dict), f"Deployed column missing: {columns}"
|
||||
|
||||
value = deployed.get(branch)
|
||||
assert isinstance(value, str) and len(value) == 8, f"Deployed value for {branch} invalid: {value}"
|
||||
return value
|
||||
|
||||
|
||||
def _pick_target_commit(env_id: str, branch: str) -> str:
|
||||
response = requests.get(f"{APP_URL}/branches", timeout=5)
|
||||
assert response.status_code == 200, f"Expected status 200 from /branches, got {response.status_code}"
|
||||
data = response.json()
|
||||
assert isinstance(data, dict), f"Expected dict from /branches, got {type(data)}"
|
||||
|
||||
env_branches = data.get(env_id, {})
|
||||
assert isinstance(env_branches, dict), f"Expected env branch map for {env_id}, got {env_branches}"
|
||||
|
||||
commits = env_branches.get(branch, [])
|
||||
assert isinstance(commits, list), f"Expected commits list for {branch}, got {commits}"
|
||||
|
||||
for commit in commits:
|
||||
if not isinstance(commit, dict):
|
||||
continue
|
||||
hexsha = commit.get("hexsha")
|
||||
if isinstance(hexsha, str) and len(hexsha) >= 8:
|
||||
return hexsha
|
||||
|
||||
raise AssertionError(f"Unable to find any commit for branch {branch}.")
|
||||
|
||||
|
||||
def _assert_deployed_hash_available(env_id: str, branch: str) -> None:
|
||||
value = _get_deployed_short_hash(env_id, branch)
|
||||
assert isinstance(value, str) and len(value) == 8
|
||||
|
||||
|
||||
class TestDeployColumnPlaywright:
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def setup_teardown(self) -> Generator[None, Any, None]:
|
||||
global _server_thread
|
||||
|
||||
def stop_and_remove_container() -> None:
|
||||
client = docker.from_env()
|
||||
try:
|
||||
container = client.containers.get(CONTAINER_NAME)
|
||||
container.stop(timeout=10)
|
||||
container.remove(force=True)
|
||||
logger.info("Container %s stopped and removed", CONTAINER_NAME)
|
||||
except Exception:
|
||||
logger.info("Container %s not found, no need to stop/remove", CONTAINER_NAME)
|
||||
|
||||
_is_port_free(APP_PORT, timeout=5.0)
|
||||
|
||||
stop_and_remove_container()
|
||||
yield None
|
||||
|
||||
if _server_thread and _server_thread.is_alive():
|
||||
_server_thread.join(timeout=2.0)
|
||||
logger.info("Waited for server thread to finish")
|
||||
|
||||
_is_port_free(APP_PORT, timeout=5.0)
|
||||
stop_and_remove_container()
|
||||
_server_thread = None
|
||||
|
||||
def test_deployed_column_updates_after_apply_changes_and_deploy(self, eventually: EventuallyFn) -> None:
|
||||
global _server_thread
|
||||
|
||||
env = nginx.test_local1
|
||||
env.state.set_branches([(TARGET_BRANCH, "HEAD")])
|
||||
env.state.set_dry(True)
|
||||
|
||||
app = App({env.id: env})
|
||||
_server_thread = threading.Thread(target=lambda: app.runWeb(APP_PORT), daemon=True)
|
||||
_server_thread.start()
|
||||
|
||||
eventually(
|
||||
lambda: assert_equal(
|
||||
requests.get(f"{APP_URL}/state", timeout=5).status_code,
|
||||
200,
|
||||
"Server did not respond with 200 OK within timeout",
|
||||
),
|
||||
timeout=20,
|
||||
interval=0.5,
|
||||
)
|
||||
|
||||
target_commit = _pick_target_commit(env.id, TARGET_BRANCH)
|
||||
|
||||
with brencher_page(APP_URL, env.id) as page:
|
||||
page.verify_dry_run_on()
|
||||
page.set_branch_desired_commit(TARGET_BRANCH, target_commit)
|
||||
page.verify_apply_changes_visible()
|
||||
page.set_apply_changes()
|
||||
|
||||
eventually(lambda: page.verify_apply_changes_hidden(), timeout=20, interval=0.5)
|
||||
|
||||
page.set_dry_run_off()
|
||||
eventually(lambda: page.verify_dry_run_off(), timeout=20, interval=0.5)
|
||||
page.click_refresh()
|
||||
|
||||
eventually(
|
||||
lambda: _assert_deployed_hash_available(env.id, TARGET_BRANCH),
|
||||
timeout=90,
|
||||
interval=1.0,
|
||||
)
|
||||
|
||||
deployed_short_hash = _get_deployed_short_hash(env.id, TARGET_BRANCH)
|
||||
|
||||
eventually(
|
||||
lambda: page.verify_branch_column_value(TARGET_BRANCH, "Deployed", deployed_short_hash),
|
||||
timeout=30,
|
||||
interval=0.5,
|
||||
)
|
||||
@@ -0,0 +1,132 @@
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
import socket
|
||||
from typing import Any, Callable, Generator, TypeVar
|
||||
|
||||
import docker
|
||||
import pytest
|
||||
import requests
|
||||
from app import App
|
||||
from .conftest import EventuallyFn, assert_equal, check_state
|
||||
from .playwright_helper import brencher_page
|
||||
from tests.configs import nginx
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
APP_PORT = 5003
|
||||
APP_URL = f"http://localhost:{APP_PORT}"
|
||||
CONTAINER_NAME = "test_plain-container"
|
||||
|
||||
# Module-level reference to server thread to ensure proper cleanup
|
||||
_server_thread: threading.Thread | None = None
|
||||
|
||||
|
||||
def _is_port_free(port: int, timeout: float = 5.0) -> bool:
|
||||
"""Check if a port is available for binding."""
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
result = sock.connect_ex(('127.0.0.1', port))
|
||||
sock.close()
|
||||
if result != 0:
|
||||
return True
|
||||
except Exception:
|
||||
return True
|
||||
time.sleep(0.1)
|
||||
return False
|
||||
|
||||
class TestDryRunPlaywright:
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def setup_teardown(self) -> Generator[None, Any, None]:
|
||||
global _server_thread
|
||||
|
||||
def stop_and_remove_container() -> None:
|
||||
client = docker.from_env()
|
||||
try:
|
||||
container = client.containers.get(CONTAINER_NAME)
|
||||
container.stop(timeout=10)
|
||||
container.remove(force=True)
|
||||
logger.info(f"Container {CONTAINER_NAME} stopped and removed")
|
||||
except Exception:
|
||||
logger.info(f"Container {CONTAINER_NAME} not found, no need to stop/remove")
|
||||
|
||||
# Ensure port is free before starting
|
||||
_is_port_free(APP_PORT, timeout=5.0)
|
||||
|
||||
stop_and_remove_container()
|
||||
yield None
|
||||
|
||||
# Clean up server thread
|
||||
if _server_thread and _server_thread.is_alive():
|
||||
_server_thread.join(timeout=2.0)
|
||||
logger.info("Waited for server thread to finish")
|
||||
|
||||
# Ensure port is free after test
|
||||
_is_port_free(APP_PORT, timeout=5.0)
|
||||
|
||||
stop_and_remove_container()
|
||||
_server_thread = None
|
||||
|
||||
|
||||
def test_dry_run_prevents_deploy_until_disabled(self, eventually: EventuallyFn) -> None:
|
||||
global _server_thread
|
||||
|
||||
# Start app with nginx_local1 with main branch pre-selected and dry mode on.
|
||||
env = nginx.test_local1
|
||||
env.state.set_branches([("test/main", "HEAD")])
|
||||
env.state.set_dry(True)
|
||||
app = App({env.id: env})
|
||||
_server_thread = threading.Thread(target=lambda: app.runWeb(APP_PORT), daemon=True)
|
||||
_server_thread.start()
|
||||
|
||||
# Wait for server to be up
|
||||
eventually(lambda: assert_equal(requests.get(f"{APP_URL}/state", timeout=5).status_code, 200, "Server did not respond with 200 OK within timeout"))
|
||||
logger.info("Server is up")
|
||||
|
||||
# Wait for the pipeline to stabilize with dry=True:
|
||||
# DockerContainerDeploy should return status="dry-run"
|
||||
with brencher_page(APP_URL, env.id) as page:
|
||||
eventually(
|
||||
lambda: check_state(APP_URL, lambda s: assert_equal(next(
|
||||
p for p in s[env.id]["pipeline"] if p["name"] == "SharedStateHolderInMemory"
|
||||
)["status"]['dry'], True, "Pipeline step status is not 'running'")),
|
||||
)
|
||||
|
||||
# page.verify_pipeline_step_status("DockerContainerDeploy", "dry-run")
|
||||
logger.info("Pipeline is in dry-run mode")
|
||||
|
||||
# Verify no real container was created
|
||||
docker_client = docker.from_env()
|
||||
real_containers = docker_client.containers.list(filters={"name": CONTAINER_NAME}, all=True)
|
||||
assert len(real_containers) == 0, "Container should NOT exist during dry run"
|
||||
|
||||
# Disable dry run from the UI.
|
||||
page.verify_dry_run_on()
|
||||
page.set_dry_run_off()
|
||||
page.verify_dry_run_off()
|
||||
page.click_refresh() # TODO Remove it
|
||||
|
||||
# Wait for pipeline to re-run and actually deploy the container
|
||||
eventually(
|
||||
lambda: check_state(APP_URL, lambda s: assert_equal(next(
|
||||
p for p in s[env.id]["pipeline"] if p["name"] =="SharedStateHolderInMemory"
|
||||
)["status"]['dry'], False, "Pipeline step status is not 'running'")),
|
||||
)
|
||||
# Wait for pipeline to re-run and actually deploy the container
|
||||
eventually(
|
||||
lambda: check_state(APP_URL, lambda s: assert_equal({p["name"] for p in s[env.id]["pipeline"] if bool(p['is_running'])}, set(), "Pipeline still running")),
|
||||
timeout=20,
|
||||
interval=1.0,
|
||||
)
|
||||
# page.verify_pipeline_step_status("DockerContainerDeploy", "running")
|
||||
logger.info("Container deployed and running after dry run disabled")
|
||||
|
||||
# Final sanity: real container must exist and be running
|
||||
containers = docker_client.containers.list(filters={"name": CONTAINER_NAME})
|
||||
assert len(containers) == 1, "Container should exist and be running after deploy"
|
||||
assert containers[0].status == "running"
|
||||
@@ -25,7 +25,7 @@ class TestGitIntegration:
|
||||
commit3 = repo_helper.create_commit(repo_helper.repo, "master", "branch2", "file3.txt", "content3",
|
||||
"Branch2 commit") # noqa: F841
|
||||
|
||||
repo_helper.env.branches = [("branch1", "HEAD")]
|
||||
repo_helper.set_desired_branches([("branch1", "HEAD")])
|
||||
result = repo_helper.checkout_merged.progress()
|
||||
|
||||
assert isinstance(result, CheckoutAndMergeResult)
|
||||
@@ -39,7 +39,7 @@ class TestGitIntegration:
|
||||
])
|
||||
assert repo_helper.git_unmerge.progress().branches == [('branch1', commit2.hexsha)], f"Invalid Unmerge result"
|
||||
|
||||
repo_helper.env.branches = [("branch2", "HEAD")]
|
||||
repo_helper.set_desired_branches([("branch2", "HEAD")])
|
||||
result = repo_helper.checkout_merged.progress()
|
||||
|
||||
assert isinstance(result, CheckoutAndMergeResult)
|
||||
@@ -66,7 +66,7 @@ class TestGitIntegration:
|
||||
"Branch2 commit") # noqa: F841
|
||||
|
||||
# Create a GitClone-like working directory
|
||||
repo_helper.env.branches = [("branch1", "HEAD"), ("branch2", "HEAD")]
|
||||
repo_helper.set_desired_branches([("branch1", "HEAD"), ("branch2", "HEAD")])
|
||||
result = repo_helper.checkout_merged.progress()
|
||||
|
||||
# Verify result with specific field checks
|
||||
@@ -99,7 +99,7 @@ class TestGitIntegration:
|
||||
commit4 = repo_helper.create_commit(repo_helper.repo, "master", "branch3", "file4.txt", "content4",
|
||||
"Branch3 commit") # noqa: F841
|
||||
|
||||
repo_helper.env.branches = [("branch1", "HEAD"), ("branch2", "HEAD"), ("branch3", "HEAD")]
|
||||
repo_helper.set_desired_branches([("branch1", "HEAD"), ("branch2", "HEAD"), ("branch3", "HEAD")])
|
||||
|
||||
result = repo_helper.checkout_merged.progress()
|
||||
|
||||
@@ -139,7 +139,7 @@ class TestGitIntegration:
|
||||
commit3 = repo_helper.create_commit(repo_helper.repo, "branch1", "branch1", "file3.txt", "content3",
|
||||
"Branch1 commit 2") # noqa: F841
|
||||
|
||||
repo_helper.env.branches = [("master", "HEAD"), ("branch1", "HEAD")]
|
||||
repo_helper.set_desired_branches([("master", "HEAD"), ("branch1", "HEAD")])
|
||||
|
||||
result = repo_helper.checkout_merged.progress()
|
||||
|
||||
@@ -165,7 +165,7 @@ class TestGitIntegration:
|
||||
"Branch2 change") # noqa: F841
|
||||
|
||||
# Create environment
|
||||
repo_helper.env.branches = [("branch1", "HEAD"), ("branch2", "HEAD")]
|
||||
repo_helper.set_desired_branches([("branch1", "HEAD"), ("branch2", "HEAD")])
|
||||
|
||||
# Expect merge conflict to be raised as BaseException
|
||||
with pytest.raises(BaseException, match="Merge conflict"):
|
||||
@@ -184,7 +184,7 @@ class TestGitIntegration:
|
||||
commit3 = repo_helper.create_commit(repo_helper.repo, "master", "branch2", "file3.txt", "content3",
|
||||
"Branch2 commit") # noqa: F841
|
||||
|
||||
repo_helper.env.branches = [("branch1", "HEAD"), ("branch2", "HEAD")]
|
||||
repo_helper.set_desired_branches([("branch1", "HEAD"), ("branch2", "HEAD")])
|
||||
result1 = repo_helper.checkout_merged.progress()
|
||||
auto_branch_name = result1.remote_branch_name
|
||||
|
||||
@@ -212,7 +212,7 @@ class TestGitIntegration:
|
||||
commit3 = repo_helper.create_commit(repo_helper.repo, "master", "branch2", "file3.txt", "content3",
|
||||
"Branch2 commit")
|
||||
|
||||
repo_helper.env.branches = []
|
||||
repo_helper.set_desired_branches([])
|
||||
|
||||
# Mock DockerSwarmCheck with version string
|
||||
version_str = f"auto-{commit2.hexsha[:8]}-{commit3.hexsha[:8]}"
|
||||
@@ -244,7 +244,7 @@ class TestGitIntegration:
|
||||
commit1 = repo_helper.create_commit(repo_helper.repo, "master", "master", "file1.txt", "content1",
|
||||
"Initial commit") # noqa: F841
|
||||
|
||||
repo_helper.env.branches = []
|
||||
repo_helper.set_desired_branches([])
|
||||
version_str = "invalid-version-format"
|
||||
repo_helper.mock_check.version = lambda: version_str # type: ignore[attr-defined]
|
||||
|
||||
@@ -262,7 +262,7 @@ class TestGitIntegration:
|
||||
commit3 = repo_helper.create_commit(repo_helper.repo, "branch1", "branch1", "file3.txt", "content3",
|
||||
"Branch1 commit 2") # noqa: F841
|
||||
|
||||
repo_helper.env.branches = []
|
||||
repo_helper.set_desired_branches([])
|
||||
|
||||
repo_helper.mock_check.version = lambda: f"auto-{commit2.hexsha[:8]}" # type: ignore[attr-defined]
|
||||
|
||||
@@ -297,7 +297,7 @@ class TestGitIntegration:
|
||||
commit5 = repo_helper.create_commit(repo_helper.repo, "branch2", "branch2", "file5.txt", "content5",
|
||||
"Branch2 commit 2")
|
||||
|
||||
repo_helper.env.branches = []
|
||||
repo_helper.set_desired_branches([])
|
||||
|
||||
repo_helper.mock_check.version = lambda: f"auto-{commit2.hexsha[:8]}-{commit4.hexsha[:8]}" # type: ignore[attr-defined]
|
||||
|
||||
@@ -333,12 +333,12 @@ class TestGitIntegration:
|
||||
commit1 = repo_helper.create_commit(repo_helper.repo, "master", "master", "file1.txt", "content1",
|
||||
"Initial commit") # noqa: F841
|
||||
|
||||
repo_helper.env.branches = []
|
||||
repo_helper.set_desired_branches([])
|
||||
|
||||
with pytest.raises(BaseException, match="Empty branches set"):
|
||||
repo_helper.checkout_merged.progress()
|
||||
|
||||
repo_helper.env.branches = [('master', 'HEAD')]
|
||||
repo_helper.set_desired_branches([('master', 'HEAD')])
|
||||
result = repo_helper.checkout_merged.progress()
|
||||
|
||||
# Verify same auto branch is used with actual value checks
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from enironment import SharedStateConflictError
|
||||
from processing import process_all_jobs
|
||||
from steps.git import GitClone
|
||||
from steps.shared_state import SharedStateHolderInGit
|
||||
from tests.test_remote_repo import RemoteRepoHelper
|
||||
|
||||
|
||||
class TestInitialBranches:
|
||||
"""Integration tests for git operations"""
|
||||
|
||||
def test_resolve_initial_branches(self, repo_helper: RemoteRepoHelper) -> None:
|
||||
_ = repo_helper.create_commit(repo_helper.repo, "master", "master", "file1.txt", "content1", "Initial commit")
|
||||
_ = repo_helper.create_commit(repo_helper.repo, "master", "branch1", "file2.txt", "content2", "Branch1 commit")
|
||||
_ = repo_helper.create_commit(repo_helper.repo, "state", "state", "blank.txt", "", "Branch1 commit")
|
||||
|
||||
clone = GitClone(url=repo_helper.remote_dir)
|
||||
clone.env = repo_helper.env
|
||||
cloneState = GitClone(url=repo_helper.remote_dir, n='state')
|
||||
cloneState.env = repo_helper.env
|
||||
resolve_step = SharedStateHolderInGit(clone, state_repo=cloneState, state_branch="state")
|
||||
resolve_step.env = repo_helper.env
|
||||
stale_token = resolve_step.progress().token
|
||||
assert isinstance(stale_token, str)
|
||||
|
||||
t = resolve_step.set_branches([("branch1", "HEAD")], expected_token=stale_token).token
|
||||
t = resolve_step.set_dry(True, expected_token=t).token
|
||||
|
||||
with pytest.raises(SharedStateConflictError, match="State token mismatch"):
|
||||
resolve_step.set_branches([("master", "HEAD")], expected_token=stale_token)
|
||||
|
||||
t = resolve_step.set_branches([("master", "HEAD")], expected_token=t).token
|
||||
|
||||
t = resolve_step.set_dry(False, expected_token=t).token
|
||||
|
||||
jpl = repo_helper.repo.git.show(f"state:{repo_helper.env.id}.json")
|
||||
pl = json.loads(jpl)
|
||||
assert pl['branches'] == [["master", "HEAD"]]
|
||||
assert pl['dry'] == False
|
||||
+148
-125
@@ -7,157 +7,180 @@ git operations in an isolated environment.
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
from typing import Dict, Tuple, List, Callable
|
||||
from typing import Dict, Tuple, List, Callable, cast
|
||||
|
||||
import git
|
||||
from enironment import Environment, AbstractStep
|
||||
|
||||
from enironment import Environment, AbstractStep, SharedStateHolder, get_step
|
||||
from steps.docker import DockerSwarmCheckResult
|
||||
from steps.git import CheckoutAndMergeResult, CheckoutMerged, GitUnmerge, GitClone, HasVersion, GitUnmergeResult
|
||||
from steps.shared_state import SharedStateHolderInMemory
|
||||
from steps.step import CachingStep # noqa: F401
|
||||
|
||||
|
||||
class MockDockerSwarmCheck(AbstractStep[Dict[str, HasVersion]]):
|
||||
"""Mock DockerSwarmCheck object for testing"""
|
||||
"""Mock DockerSwarmCheck object for testing"""
|
||||
|
||||
def __init__(self, version: Callable[[], str]) -> None:
|
||||
super().__init__()
|
||||
self.version = version
|
||||
self.stack_name = "test_stack"
|
||||
def __init__(self, version: Callable[[], str]) -> None:
|
||||
super().__init__()
|
||||
self.version = version
|
||||
self.stack_name = "test_stack"
|
||||
|
||||
def progress(self) -> Dict[str, HasVersion]:
|
||||
return {
|
||||
"service1": DockerSwarmCheckResult(
|
||||
name="service1",
|
||||
version=self.version(),
|
||||
image="test:latest",
|
||||
stack="test-stack"
|
||||
)
|
||||
}
|
||||
def progress(self) -> Dict[str, HasVersion]:
|
||||
return {
|
||||
"service1": DockerSwarmCheckResult(
|
||||
name="service1",
|
||||
version=self.version(),
|
||||
image="test:latest",
|
||||
stack="test-stack"
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
class RemoteRepoHelper:
|
||||
"""Helper class for creating and managing test git repositories"""
|
||||
env: Environment
|
||||
"""Helper class for creating and managing test git repositories"""
|
||||
env: Environment
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.remote_dir = tempfile.mkdtemp(prefix="test_remote_")
|
||||
self.local_dir = tempfile.mkdtemp(prefix="test_local_")
|
||||
self.repo = git.Repo.init(self.remote_dir, bare=False)
|
||||
# Configure git user
|
||||
with self.repo.config_writer() as cw:
|
||||
cw.set_value("user", "email", "test@example.com")
|
||||
cw.set_value("user", "name", "Test User")
|
||||
git_clone = GitClone(repo_path=self.local_dir)
|
||||
checkout_merged = CheckoutMerged(
|
||||
wd=git_clone,
|
||||
git_user_email="test@example.com",
|
||||
git_user_name="Test User",
|
||||
push=False,
|
||||
)
|
||||
mock_check = MockDockerSwarmCheck(lambda: f"auto-{self.checkout_merged.progress().version}")
|
||||
git_unmerge = GitUnmerge(
|
||||
wd=git_clone,
|
||||
check=mock_check,
|
||||
)
|
||||
self.env = Environment(
|
||||
id="test1",
|
||||
branches=[],
|
||||
dry=False,
|
||||
repo=self.remote_dir,
|
||||
pipeline=[
|
||||
git_clone,
|
||||
checkout_merged,
|
||||
mock_check,
|
||||
git_unmerge
|
||||
]
|
||||
)
|
||||
def __init__(self) -> None:
|
||||
self.remote_dir = tempfile.mkdtemp(prefix="test_remote_")
|
||||
self.local_dir = tempfile.mkdtemp(prefix="test_local_")
|
||||
self.repo = git.Repo.init(self.remote_dir, bare=False)
|
||||
# Configure git user
|
||||
with self.repo.config_writer() as cw:
|
||||
cw.set_value("user", "email", "test@example.com")
|
||||
cw.set_value("user", "name", "Test User")
|
||||
git_clone = GitClone(url=self.remote_dir, repo_path=self.local_dir)
|
||||
mock_check = MockDockerSwarmCheck(lambda: f"auto-{self.checkout_merged.progress().version}")
|
||||
git_unmerge = GitUnmerge(
|
||||
wd=git_clone,
|
||||
check=mock_check,
|
||||
)
|
||||
state = SharedStateHolderInMemory(unmerge=None)
|
||||
checkout_merged = CheckoutMerged(
|
||||
wd=git_clone,
|
||||
desired_branches=state,
|
||||
git_user_email="test@example.com",
|
||||
git_user_name="Test User",
|
||||
push=False,
|
||||
)
|
||||
self.env = Environment(
|
||||
id="test1",
|
||||
state=state,
|
||||
pipeline=[
|
||||
git_clone,
|
||||
state,
|
||||
checkout_merged,
|
||||
mock_check,
|
||||
git_unmerge
|
||||
]
|
||||
)
|
||||
|
||||
@property
|
||||
def checkout_merged(self) -> AbstractStep[CheckoutAndMergeResult]:
|
||||
return [i for i in self.env.pipeline if
|
||||
isinstance(i, CheckoutMerged) or (isinstance(i, CachingStep) and isinstance(i.step, CheckoutMerged))][0]
|
||||
@property
|
||||
def checkout_merged(self) -> AbstractStep[CheckoutAndMergeResult]:
|
||||
return get_step(self.env.pipeline, CheckoutMerged)
|
||||
|
||||
@property
|
||||
def git_clone(self) -> AbstractStep[str]:
|
||||
return [i for i in self.env.pipeline if
|
||||
isinstance(i, GitClone) or (isinstance(i, CachingStep) and isinstance(i.step, GitClone))][0]
|
||||
@property
|
||||
def resolve_initial_branches(self) -> SharedStateHolder:
|
||||
return self.env.state
|
||||
|
||||
@property
|
||||
def git_unmerge(self) -> AbstractStep[GitUnmergeResult]:
|
||||
return [i for i in self.env.pipeline if
|
||||
isinstance(i, GitUnmerge) or (isinstance(i, CachingStep) and isinstance(i.step, GitUnmerge))][0]
|
||||
def set_desired_branches(self, branches: List[Tuple[str, str]]) -> None:
|
||||
step: SharedStateHolder = get_step(self.env.pipeline, SharedStateHolder) # type: ignore[type-abstract]
|
||||
if isinstance(step, CachingStep) and isinstance(step._step, SharedStateHolder):
|
||||
step._step.set_branches(branches)
|
||||
step.reset()
|
||||
else:
|
||||
step.set_branches(branches)
|
||||
|
||||
@property
|
||||
def mock_check(self) -> AbstractStep[Dict[str, HasVersion]]:
|
||||
return [i for i in self.env.pipeline if isinstance(i, MockDockerSwarmCheck) or (
|
||||
isinstance(i, CachingStep) and isinstance(i.step, MockDockerSwarmCheck))][0]
|
||||
def get_desired_branches(self) -> List[Tuple[str, str]]:
|
||||
return self.resolve_initial_branches.progress().branches
|
||||
|
||||
def teardown(self) -> None:
|
||||
"""Clean up temporary directories"""
|
||||
if self.remote_dir:
|
||||
shutil.rmtree(self.remote_dir, ignore_errors=True)
|
||||
if self.local_dir:
|
||||
shutil.rmtree(self.local_dir, ignore_errors=True)
|
||||
@property
|
||||
def git_clone(self) -> AbstractStep[str]:
|
||||
return get_step(self.env.pipeline, GitClone)
|
||||
|
||||
def create_commit(self, repo: git.Repo, from_branch: str, to_branch: str, filename: str, content: str,
|
||||
message: str) -> git.Commit:
|
||||
"""Create a commit in the repository
|
||||
@property
|
||||
def git_unmerge(self) -> AbstractStep[GitUnmergeResult]:
|
||||
return get_step(self.env.pipeline, GitUnmerge)
|
||||
|
||||
If to_branch doesn't exist, it will be created from from_branch and checked out.
|
||||
"""
|
||||
if from_branch in [head.name for head in repo.heads]:
|
||||
repo.heads[from_branch].checkout()
|
||||
if from_branch != to_branch:
|
||||
if to_branch not in [head.name for head in repo.heads]:
|
||||
new_branch = repo.create_head(to_branch)
|
||||
new_branch.checkout()
|
||||
else:
|
||||
raise BaseException("Target branch already exists.")
|
||||
@property
|
||||
def mock_check(self) -> AbstractStep[Dict[str, HasVersion]]:
|
||||
return get_step(self.env.pipeline, MockDockerSwarmCheck)
|
||||
|
||||
# Create commit
|
||||
file_path = os.path.join(repo.working_dir, filename)
|
||||
with open(file_path, 'w') as f:
|
||||
f.write(content)
|
||||
repo.index.add([filename])
|
||||
return repo.index.commit(message)
|
||||
def teardown(self) -> None:
|
||||
"""Clean up temporary directories"""
|
||||
if self.remote_dir:
|
||||
shutil.rmtree(self.remote_dir, ignore_errors=True)
|
||||
if self.local_dir:
|
||||
shutil.rmtree(self.local_dir, ignore_errors=True)
|
||||
|
||||
def verify_working_directory_files(self, expected_files: List[Tuple[str, str]]) -> None:
|
||||
"""Verify that working directory contains exactly the expected files with correct content.
|
||||
def create_commit(self, repo: git.Repo, from_branch: str, to_branch: str, filename: str, content: str,
|
||||
message: str) -> git.Commit:
|
||||
"""Create a commit in the repository
|
||||
|
||||
Args:
|
||||
directory: Path to the working directory to verify
|
||||
expected_files: List of tuples (filename, expected_content)
|
||||
"""
|
||||
files_in_wd = set(os.listdir(self.local_dir)) - {'.git'}
|
||||
expected_filenames = {filename for filename, _ in expected_files}
|
||||
assert files_in_wd == expected_filenames, f"Expected files {expected_filenames}, got {files_in_wd}"
|
||||
If to_branch doesn't exist, it will be created from from_branch and checked out.
|
||||
"""
|
||||
if from_branch not in [head.name for head in repo.heads]:
|
||||
# Create initial commit without parents
|
||||
file_path = os.path.join(repo.working_dir, filename)
|
||||
with open(file_path, 'w') as f:
|
||||
f.write(content)
|
||||
repo.index.add([filename])
|
||||
repo.index.commit(message)
|
||||
repo.create_head(from_branch)
|
||||
return repo.head.commit
|
||||
else:
|
||||
repo.heads[from_branch].checkout()
|
||||
if from_branch != to_branch:
|
||||
if to_branch not in [head.name for head in repo.heads]:
|
||||
new_branch = repo.create_head(to_branch)
|
||||
new_branch.checkout()
|
||||
else:
|
||||
raise BaseException("Target branch already exists.")
|
||||
|
||||
for filename, expected_content in expected_files:
|
||||
with open(os.path.join(self.local_dir, filename)) as f:
|
||||
actual_content = f.read()
|
||||
assert actual_content == expected_content, f"{filename} has incorrect content. Expected '{expected_content}', got '{actual_content}'"
|
||||
# Create commit
|
||||
file_path = os.path.join(repo.working_dir, filename)
|
||||
with open(file_path, 'w') as f:
|
||||
f.write(content)
|
||||
repo.index.add([filename])
|
||||
return repo.index.commit(message)
|
||||
|
||||
def _safe_git_log(self, repo_path: str, title: str) -> str:
|
||||
def _as_text(value: object) -> str:
|
||||
if isinstance(value, bytes):
|
||||
return value.decode(errors='replace')
|
||||
return str(value)
|
||||
def verify_working_directory_files(self, expected_files: List[Tuple[str, str]]) -> None:
|
||||
"""Verify that working directory contains exactly the expected files with correct content.
|
||||
|
||||
repo = git.Repo(repo_path)
|
||||
commits = list(repo.iter_commits("--all", max_count=30))
|
||||
refs = [
|
||||
f"{_as_text(it.commit)[:10]}:{_as_text(it.name)}, is_remote:{it.is_remote()}"
|
||||
for it in repo.refs
|
||||
]
|
||||
lines = [
|
||||
f"{_as_text(commit.hexsha)[:10]} (p{[_as_text(it.hexsha)[:10] for it in commit.parents]}): {_as_text(commit.message).splitlines()[0]}"
|
||||
for commit in commits
|
||||
]
|
||||
return f"[{title}] git log ({repo_path})\n{"\n".join(lines)}\nrefs:\n{"\n".join(refs)}"
|
||||
Args:
|
||||
directory: Path to the working directory to verify
|
||||
expected_files: List of tuples (filename, expected_content)
|
||||
"""
|
||||
files_in_wd = set(os.listdir(self.local_dir)) - {'.git'}
|
||||
expected_filenames = {filename for filename, _ in expected_files}
|
||||
assert files_in_wd == expected_filenames, f"Expected files {expected_filenames}, got {files_in_wd}"
|
||||
|
||||
def print_git_logs(self) -> None:
|
||||
"""Print remote and local repository logs for debugging failed tests."""
|
||||
print(f"Version: {git.__version__}")
|
||||
for filename, expected_content in expected_files:
|
||||
with open(os.path.join(self.local_dir, filename)) as f:
|
||||
actual_content = f.read()
|
||||
assert actual_content == expected_content, f"{filename} has incorrect content. Expected '{expected_content}', got '{actual_content}'"
|
||||
|
||||
print(self._safe_git_log(self.remote_dir, "remote"))
|
||||
print(self._safe_git_log(self.local_dir, "local"))
|
||||
def _safe_git_log(self, repo_path: str, title: str) -> str:
|
||||
def _as_text(value: object) -> str:
|
||||
if isinstance(value, bytes):
|
||||
return value.decode(errors='replace')
|
||||
return str(value)
|
||||
|
||||
repo = git.Repo(repo_path)
|
||||
commits = list(repo.iter_commits("--all", max_count=30))
|
||||
refs = [
|
||||
f"{_as_text(it.commit)[:10]}:{_as_text(it.name)}, is_remote:{it.is_remote()}"
|
||||
for it in repo.refs
|
||||
]
|
||||
lines = [
|
||||
f"{_as_text(commit.hexsha)[:10]} (p{[_as_text(it.hexsha)[:10] for it in commit.parents]}): {_as_text(commit.message).splitlines()[0]}"
|
||||
for commit in commits
|
||||
]
|
||||
return f"[{title}] git log ({repo_path})\n{"\n".join(lines)}\nrefs:\n{"\n".join(refs)}"
|
||||
|
||||
def print_git_logs(self) -> None:
|
||||
"""Print remote and local repository logs for debugging failed tests."""
|
||||
print(f"Version: {git.__version__}")
|
||||
|
||||
print(self._safe_git_log(self.remote_dir, "remote"))
|
||||
print(self._safe_git_log(self.local_dir, "local"))
|
||||
|
||||
Reference in New Issue
Block a user