stopped on building image

This commit is contained in:
2025-08-07 17:26:12 +02:00
parent eb09da9f30
commit 190c461e1f
13 changed files with 202 additions and 132 deletions
+1
View File
@@ -15,4 +15,5 @@
id: str
branches: List[Tuple[str, str]]
state: str # 'Active' or 'Pause'
pipeline: Dict[str, Any]
+4
View File
@@ -75,3 +75,7 @@ All steps must inherit from BaseStep and implement idempotent behavior. This mea
- Ensures idempotency by checking if the image with the same tag already exists in the repository before building/pushing
- If all images exist and match, skips build and push
- If failed, save result to <output>
## DockerSwarmDeploy
### Logic
Deploys to swarm service described in specified docker-compose.yaml
+24 -7
View File
@@ -84,10 +84,30 @@ class BranchesNamespace(Namespace):
def on_update(self, data):
emit('branches', branches)
def get_envs_to_emit():
env_dtos = []
for e, p in environments:
env = asdict(e)
res = []
for r in p:
if isinstance(r.result_obj, BaseException):
res.append({
"name":r.name,
"status": str(r.result_obj)
})
else:
res.append({
"name":r.name,
"status": r.result_obj
})
env_dtos.append((env, res))
return env_dtos
class EnvironmentNamespace(Namespace):
def on_connect(self, auth=None):
env_dtos = [asdict(e) for e, _ in environments]
emit('environments', env_dtos)
emit('environments', get_envs_to_emit())
def on_update(self, data):
# Accept environment update from UI
# Update environments and broadcast
@@ -95,8 +115,7 @@ class EnvironmentNamespace(Namespace):
if e.id == data.get('id'):
e.state = data.get('state', e.state)
e.branches = data.get('branches', e.branches)
env_dtos = [asdict(e) for e, _ in environments]
emit('environments', env_dtos, broadcast=True)
emit('environments', get_envs_to_emit(), broadcast=True)
class ErrorsNamespace(Namespace):
def on_connect(self, auth=None):
@@ -124,9 +143,7 @@ if __name__ == '__main__':
def processing_thread():
while True:
import processing
processing.do_job(environments, lambda e: socketio.emit('error', e))
env_dtos = [asdict(e) for e, _ in environments]
socketio.emit('environments', env_dtos)
processing.do_job(environments, lambda: socketio.emit('environments', get_envs_to_emit()))
time.sleep(10)
processing = threading.Thread(target=processing_thread)
processing.daemon = True
+17 -4
View File
@@ -1,7 +1,7 @@
from steps.git import GitClone
from steps.git import CheckoutMerged
from steps.docker import DockerComposeBuild
from steps.docker import DockerComposeDeploy
from steps.docker import DockerSwarmDeploy
from enironment import Environment
from typing import List, Dict, Any, Optional, Tuple
from steps.step import AbstractStep
@@ -16,9 +16,22 @@ env = Environment(
def create_pipeline(env: Environment) -> List[AbstractStep]:
clone = GitClone(env)
checkoutMerged = CheckoutMerged(clone, env.branches)
buildDocker = DockerComposeBuild() #wd=lambda: clone.path, version=lambda: checkoutMerged.version)
deployDocker = DockerComposeDeploy() #version=lambda: checkoutMerged.version)
checkoutMerged = CheckoutMerged(clone, env.branches, env=env)
buildDocker = DockerComposeBuild(clone,
docker_repo_username = "",
docker_repo_password = "",
docker_compose_path = "docker-compose.yml",
docker_repo_url="https://registry.rudolf.keenetic.link",
envs = lambda: { "version": "auto-" + checkoutMerged.result[1][0:5] },
env=env
)
deployDocker = DockerSwarmDeploy(
wd=clone,
envs = lambda: { "version": "auto-" + checkoutMerged.result[1][0:5] },
stack_name = "brencher",
docker_compose_path = "docker-compose.yml",
env=env,
)
return [
clone,
checkoutMerged,
-1
View File
@@ -1,6 +1,5 @@
from dataclasses import dataclass, asdict, field
from typing import List, Dict, Any, Optional, Tuple, Callable
from steps.step import AbstractStep
@dataclass
class Environment:
+7 -4
View File
@@ -5,18 +5,21 @@ import logging
logger = logging.getLogger(__name__)
def do_job(environemnts: List[Tuple[Environment, List[AbstractStep]]], onerror: Callable[[str], None]):
def do_job(
environemnts: List[Tuple[Environment, List[AbstractStep]]],
onupdate: Callable[[], None]
):
for env, pipe in environemnts:
try:
if env.state != 'Active':
continue
for step in pipe:
if isinstance(step, AbstractStep):
step.progress()
step.do_job()
onupdate()
except Exception as e:
error_msg = f"Error processing release {env.id}: {str(e)}"
logger.error(error_msg)
onerror(error_msg)
onupdate()
-15
View File
@@ -1,15 +0,0 @@
from dataclasses import dataclass
from typing import Optional
@dataclass
class CheckoutMergedResult:
branch_name: Optional[str] = None
commit_hash: Optional[str] = None
success: bool = True
error_message: Optional[str] = None
class CheckoutMerged:
def process(self, clone_result, branches):
# Implement the logic for checking out merged branches here
# Return a CheckoutMergedResult instance
pass
+64 -51
View File
@@ -3,20 +3,31 @@ import subprocess
import logging
import docker
import yaml
from typing import List, Optional
from typing import List, Optional, Dict, Callable, Any
from dataclasses import dataclass
from steps.step import AbstractStep
from steps.git import GitClone
import re
logger = logging.getLogger(__name__)
@dataclass
class DockerBuildResult:
images_built: List[str]
success: bool
error_message: Optional[str] = None
class DockerComposeBuild(AbstractStep[List[str]]):
def __init__(self,
wd: GitClone,
docker_repo_username:str,
docker_repo_password:str,
docker_compose_path:str,
docker_repo_url: str,
envs: Callable[[], Dict[str, str]], **kwargs):
super().__init__(**kwargs)
self.wd = wd
self.envs = envs
self.docker_repo_username = docker_repo_username
self.docker_repo_password = docker_repo_password
self.docker_compose_path = docker_compose_path
self.docker_repo_url = docker_repo_url
class DockerComposeBuild(AbstractStep):
def progress(self) -> None:
def progress(self) -> List[str]:
# def process(self, docker_compose_path: str, docker_repo_url: str, docker_repo_username: str, docker_repo_password: str) -> DockerBuildResult:
"""
Build and push Docker images defined in a docker-compose file.
@@ -25,11 +36,13 @@ class DockerComposeBuild(AbstractStep):
try:
# Authenticate to docker repo
client = docker.from_env()
client.login(username=docker_repo_username, password=docker_repo_password, registry=docker_repo_url)
client.login(username=self.docker_repo_username, password=self.docker_repo_password, registry=self.docker_repo_url)
env = self.envs()
# Parse docker-compose file
with open(docker_compose_path, 'r') as f:
compose = yaml.safe_load(f)
with open(self.docker_compose_path, 'r') as f:
content = f.read()
content = re.sub(r'\$\{([^}]+)\}', lambda m: env[m.group(1)], content)
compose = yaml.safe_load(content)
services = compose.get('services', {})
for name, svc in services.items():
@@ -52,49 +65,49 @@ class DockerComposeBuild(AbstractStep):
for line in client.images.push(image, stream=True, decode=True):
logger.debug(line)
images_built.append(image)
return DockerBuildResult(images_built=images_built, success=True)
return images_built
except Exception as e:
logger.error(f"DockerBuild failed: {e}")
return DockerBuildResult(images_built=images_built, success=False, error_message=str(e))
raise e
class DockerComposeDeploy(AbstractStep):
def progress(self) -> None:
# def process(self, docker_compose_path: str, docker_repo_url: str, docker_repo_username: str, docker_repo_password: str) -> DockerBuildResult:
class DockerSwarmDeploy(AbstractStep):
def __init__(self,
wd: GitClone,
docker_compose_path: str,
envs: Callable[[], Dict[str, str]],
stack_name: str,
**kwargs):
super().__init__(**kwargs)
self.wd = wd
self.envs = envs
self.docker_compose_path = docker_compose_path
self.stack_name = stack_name
def progress(self) -> None:
"""
Build and push Docker images defined in a docker-compose file.
Deploys to Docker Swarm using the specified docker-compose.yaml.
"""
images_built = []
try:
# Authenticate to docker repo
client = docker.from_env()
client.login(username=docker_repo_username, password=docker_repo_password, registry=docker_repo_url)
env = self.envs()
# Prepare docker-compose file with env substitution
with open(self.docker_compose_path, 'r') as f:
content = f.read()
content = re.sub(r'\$\{([^}]+)\}', lambda m: env.get(m.group(1), ""), content)
tmp_compose_path = self.docker_compose_path + ".tmp"
with open(tmp_compose_path, 'w') as f:
f.write(content)
# Parse docker-compose file
with open(docker_compose_path, 'r') as f:
compose = yaml.safe_load(f)
services = compose.get('services', {})
for name, svc in services.items():
build_ctx = svc.get('build')
image = svc.get('image')
if not build_ctx or not image:
continue
# Check if image exists in remote repo
try:
client.images.pull(image)
logger.info(f"Image {image} already exists in repo, skipping build.")
continue
except Exception:
pass
# Build image
logger.info(f"Building image {image} from {build_ctx}")
client.images.build(path=build_ctx, tag=image)
# Push image
logger.info(f"Pushing image {image}")
for line in client.images.push(image, stream=True, decode=True):
logger.debug(line)
images_built.append(image)
return DockerBuildResult(images_built=images_built, success=True)
# Deploy stack to Docker Swarm
cmd = [
"docker", "stack", "deploy",
"-c", tmp_compose_path,
self.stack_name
]
logger.info(f"Deploying stack '{self.stack_name}' using {tmp_compose_path}")
result = subprocess.run(cmd, capture_output=True, text=True)
if result.returncode != 0:
logger.error(f"Stack deploy failed: {result.stderr}")
raise RuntimeError(f"Stack deploy failed: {result.stderr}")
logger.info(f"Stack deployed successfully: {result.stdout}")
except Exception as e:
logger.error(f"DockerBuild failed: {e}")
return DockerBuildResult(images_built=images_built, success=False, error_message=str(e))
logger.error(f"DockerSwarmDeploy failed: {e}")
raise
+25 -24
View File
@@ -9,12 +9,11 @@ from enironment import Environment
logger = logging.getLogger(__name__)
@dataclass
class GitClone(AbstractStep):
env: Environment = field(metadata={"ignore_json":True})
result: Union[str, BaseException] = BaseException("No result yet")
class GitClone(AbstractStep[str]):
def __init__(self, env: Environment, **kwargs):
super().__init__(env, **kwargs)
def progress(self) -> None:
def progress(self) -> str | BaseException:
try:
# Create temporary directory for the clone
@@ -24,29 +23,34 @@ class GitClone(AbstractStep):
logger.info(f"Cloning repository {self.env.repo} to {temp_dir}")
git.Repo.clone_from(self.env.repo, temp_dir)
self.result = temp_dir
return temp_dir
except Exception as e:
self.result = e
return e
import hashlib
@dataclass
class CheckoutMerged(AbstractStep):
class CheckoutMerged(AbstractStep[Tuple[str, str]]):
wd: GitClone
branches: List[str]
result: Union[Tuple[str, str], BaseException] = BaseException("No result yet")
def progress(self) -> None:
if self.wd.result is BaseException:
self.result = self.wd.result
return
def __init__(self, wd: GitClone, branches: List[str], **kwargs):
super().__init__(**kwargs)
self.wd = wd
self.branches = branches
repo_path = self.wd.result
def progress(self) -> Tuple[str, str] | BaseException:
if self.wd.result_obj is BaseException:
return self.wd.result_obj
repo_path = self.wd.result_obj
if not isinstance(repo_path, str):
self.result = repo_path # propagate the error (likely an exception)
return
return BaseException(f"Unknown repo path {repo_path}")
if len(self.branches) == 0:
return BaseException(f"Empty branches set")
try:
# Open the repository
repo = git.Repo(repo_path)
@@ -66,8 +70,7 @@ class CheckoutMerged(AbstractStep):
repo.git.checkout(auto_branch_name)
commit_hash = repo.head.commit.hexsha
self.result = (auto_branch_name, commit_hash)
return
return (auto_branch_name, commit_hash)
# Auto branch doesn't exist, create it
logger.info(f"Creating new auto branch: {auto_branch_name}")
@@ -92,8 +95,7 @@ class CheckoutMerged(AbstractStep):
error_message = f"Merge conflict when merging {branch}: {str(e)}"
logger.error(error_message)
self.result = BaseException(error_message)
return
return BaseException(error_message)
# Create and push the auto branch
repo.git.branch('-f', auto_branch_name)
@@ -101,10 +103,9 @@ class CheckoutMerged(AbstractStep):
commit_hash = repo.head.commit.hexsha
repo.git.push('-f', 'origin', auto_branch_name)
self.result = (auto_branch_name,commit_hash)
return (auto_branch_name,commit_hash)
except Exception as e:
error_message = f"Failed to create merged branch: {str(e)}"
logger.error(error_message)
self.result = BaseException(error_message)
return
return BaseException(error_message)
+37 -2
View File
@@ -1,7 +1,42 @@
from abc import ABC, abstractmethod
from typing import List, Optional, Union, Tuple, Any
from typing import TypeVar, Generic
from dataclasses import dataclass, asdict, field
from enironment import Environment
import logging
class AbstractStep(ABC):
logger = logging.getLogger(__name__)
T = TypeVar('T')
class AbstractStep(ABC, Generic[T]):
env: Environment
name: str
_result: T | BaseException = BaseException("No result yet")
def __init__(self, env: Environment, n: str | None = None) -> None:
if n is None:
n = self.__class__.__name__
self.name = n
self.env = env
@property
def result_obj(self) -> Union[T, BaseException]:
return self._result
@property
def result(self) -> T:
if isinstance(self._result, BaseException):
raise self._result
return self._result
def do_job(self):
try:
self._result = self.progress()
except Exception as e:
logger.error(f"Job failed: {e}")
self._result = e
@abstractmethod
def progress(self):
def progress(self) -> T | BaseException:
pass
+1 -1
View File
@@ -2,7 +2,7 @@ version: '3.8'
services:
brencher-backend:
build: .
image: registry.rudolf.keenetic.link/brencher:latest
image: registry.rudolf.keenetic.link/brencher:${version}
ports:
- "5000:5000"
environment:
+13 -18
View File
@@ -3,30 +3,25 @@
## Branch List
- Display list of branches in all git repositories
- Each branch has a checkbox (selected or unselected)
- Branch checkbox should be checked if this branch present in EnvironmentDto
## Branch Refresh
- Branches are fetched from server on startup and refreshed every 5 minutes
## Communication
## Environment Editing
- EnvironmentDto properties editable by user:
branches
state
## Environment State
- On each update for EnvironmentDto render pipeline on right conlumn on UI
## Communication and errors
- UI communicates with backend using websocket with class-based mapping
- UI receives environment updates via websocket
- UI receives errors via websocket to display
## Error Handling
- If request fails, show error message in bar
- If backend response is not 2xx, display error with message from response
## Environment Editing
- Environment is editable by user
- EnvironmentDto contains list of branches
## Environment State
- Display list of jobs from EnvironmentState
## DTOs
- EnvironmentDto: { branches: [string] }
- EnvironmentStateDto: { jobs: [JobDto] }
## Notes
- UI should handle websocket events for environment updates and errors
- UI should provide feedback for failed requests and backend errors
- If request fails, show error message in bar
- If backend response is not 2xx, display error with message from response
+9 -5
View File
@@ -24,6 +24,7 @@ let defaultState = 'Active';
let environment = null;
let jobs = [];
let branchStates = {};
let wsBranches = null;
let wsEnv = null;
let wsErr = null;
@@ -88,15 +89,17 @@ function renderReleases() {
function renderJobs() {
const jobsList = document.getElementById('jobs-list');
if (!jobsList) return;
if (!jobs.length) {
if (!environment) return;
if (!environment.length) {
jobsList.innerHTML = '<p class="loading">No jobs found.</p>';
return;
}
jobsList.innerHTML = jobs.map(job => `
<div class="job-item">
jobsList.innerHTML = environment[0][1].map(job => {
return `<div class="job-item">
<strong>${job.name}</strong> - ${job.status}
</div>
`).join('');
</div>`
}).join('');
}
stateSelector.onchange = (e) => {
@@ -123,6 +126,7 @@ function setupSocketIO() {
wsEnv.on('environments', (data) => {
environment = data;
renderBranches();
renderJobs()
showStatus('Environment updated via Socket.IO.');
});