stopped on building image
This commit is contained in:
@@ -15,4 +15,5 @@
|
||||
id: str
|
||||
branches: List[Tuple[str, str]]
|
||||
state: str # 'Active' or 'Pause'
|
||||
pipeline: Dict[str, Any]
|
||||
|
||||
|
||||
@@ -75,3 +75,7 @@ All steps must inherit from BaseStep and implement idempotent behavior. This mea
|
||||
- Ensures idempotency by checking if the image with the same tag already exists in the repository before building/pushing
|
||||
- If all images exist and match, skips build and push
|
||||
- If failed, save result to <output>
|
||||
|
||||
## DockerSwarmDeploy
|
||||
### Logic
|
||||
Deploys to swarm service described in specified docker-compose.yaml
|
||||
+24
-7
@@ -84,10 +84,30 @@ class BranchesNamespace(Namespace):
|
||||
def on_update(self, data):
|
||||
emit('branches', branches)
|
||||
|
||||
def get_envs_to_emit():
|
||||
env_dtos = []
|
||||
for e, p in environments:
|
||||
env = asdict(e)
|
||||
res = []
|
||||
for r in p:
|
||||
if isinstance(r.result_obj, BaseException):
|
||||
res.append({
|
||||
"name":r.name,
|
||||
"status": str(r.result_obj)
|
||||
})
|
||||
else:
|
||||
res.append({
|
||||
"name":r.name,
|
||||
"status": r.result_obj
|
||||
})
|
||||
env_dtos.append((env, res))
|
||||
return env_dtos
|
||||
|
||||
class EnvironmentNamespace(Namespace):
|
||||
|
||||
def on_connect(self, auth=None):
|
||||
env_dtos = [asdict(e) for e, _ in environments]
|
||||
emit('environments', env_dtos)
|
||||
emit('environments', get_envs_to_emit())
|
||||
|
||||
def on_update(self, data):
|
||||
# Accept environment update from UI
|
||||
# Update environments and broadcast
|
||||
@@ -95,8 +115,7 @@ class EnvironmentNamespace(Namespace):
|
||||
if e.id == data.get('id'):
|
||||
e.state = data.get('state', e.state)
|
||||
e.branches = data.get('branches', e.branches)
|
||||
env_dtos = [asdict(e) for e, _ in environments]
|
||||
emit('environments', env_dtos, broadcast=True)
|
||||
emit('environments', get_envs_to_emit(), broadcast=True)
|
||||
|
||||
class ErrorsNamespace(Namespace):
|
||||
def on_connect(self, auth=None):
|
||||
@@ -124,9 +143,7 @@ if __name__ == '__main__':
|
||||
def processing_thread():
|
||||
while True:
|
||||
import processing
|
||||
processing.do_job(environments, lambda e: socketio.emit('error', e))
|
||||
env_dtos = [asdict(e) for e, _ in environments]
|
||||
socketio.emit('environments', env_dtos)
|
||||
processing.do_job(environments, lambda: socketio.emit('environments', get_envs_to_emit()))
|
||||
time.sleep(10)
|
||||
processing = threading.Thread(target=processing_thread)
|
||||
processing.daemon = True
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from steps.git import GitClone
|
||||
from steps.git import CheckoutMerged
|
||||
from steps.docker import DockerComposeBuild
|
||||
from steps.docker import DockerComposeDeploy
|
||||
from steps.docker import DockerSwarmDeploy
|
||||
from enironment import Environment
|
||||
from typing import List, Dict, Any, Optional, Tuple
|
||||
from steps.step import AbstractStep
|
||||
@@ -16,9 +16,22 @@ env = Environment(
|
||||
|
||||
def create_pipeline(env: Environment) -> List[AbstractStep]:
|
||||
clone = GitClone(env)
|
||||
checkoutMerged = CheckoutMerged(clone, env.branches)
|
||||
buildDocker = DockerComposeBuild() #wd=lambda: clone.path, version=lambda: checkoutMerged.version)
|
||||
deployDocker = DockerComposeDeploy() #version=lambda: checkoutMerged.version)
|
||||
checkoutMerged = CheckoutMerged(clone, env.branches, env=env)
|
||||
buildDocker = DockerComposeBuild(clone,
|
||||
docker_repo_username = "",
|
||||
docker_repo_password = "",
|
||||
docker_compose_path = "docker-compose.yml",
|
||||
docker_repo_url="https://registry.rudolf.keenetic.link",
|
||||
envs = lambda: { "version": "auto-" + checkoutMerged.result[1][0:5] },
|
||||
env=env
|
||||
)
|
||||
deployDocker = DockerSwarmDeploy(
|
||||
wd=clone,
|
||||
envs = lambda: { "version": "auto-" + checkoutMerged.result[1][0:5] },
|
||||
stack_name = "brencher",
|
||||
docker_compose_path = "docker-compose.yml",
|
||||
env=env,
|
||||
)
|
||||
return [
|
||||
clone,
|
||||
checkoutMerged,
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
from dataclasses import dataclass, asdict, field
|
||||
from typing import List, Dict, Any, Optional, Tuple, Callable
|
||||
from steps.step import AbstractStep
|
||||
|
||||
@dataclass
|
||||
class Environment:
|
||||
|
||||
@@ -5,18 +5,21 @@ import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def do_job(environemnts: List[Tuple[Environment, List[AbstractStep]]], onerror: Callable[[str], None]):
|
||||
def do_job(
|
||||
environemnts: List[Tuple[Environment, List[AbstractStep]]],
|
||||
onupdate: Callable[[], None]
|
||||
):
|
||||
for env, pipe in environemnts:
|
||||
try:
|
||||
if env.state != 'Active':
|
||||
continue
|
||||
for step in pipe:
|
||||
if isinstance(step, AbstractStep):
|
||||
step.progress()
|
||||
|
||||
step.do_job()
|
||||
onupdate()
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"Error processing release {env.id}: {str(e)}"
|
||||
logger.error(error_msg)
|
||||
onerror(error_msg)
|
||||
onupdate()
|
||||
|
||||
|
||||
@@ -1,15 +0,0 @@
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional
|
||||
|
||||
@dataclass
|
||||
class CheckoutMergedResult:
|
||||
branch_name: Optional[str] = None
|
||||
commit_hash: Optional[str] = None
|
||||
success: bool = True
|
||||
error_message: Optional[str] = None
|
||||
|
||||
class CheckoutMerged:
|
||||
def process(self, clone_result, branches):
|
||||
# Implement the logic for checking out merged branches here
|
||||
# Return a CheckoutMergedResult instance
|
||||
pass
|
||||
+64
-51
@@ -3,20 +3,31 @@ import subprocess
|
||||
import logging
|
||||
import docker
|
||||
import yaml
|
||||
from typing import List, Optional
|
||||
from typing import List, Optional, Dict, Callable, Any
|
||||
from dataclasses import dataclass
|
||||
from steps.step import AbstractStep
|
||||
from steps.git import GitClone
|
||||
import re
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@dataclass
|
||||
class DockerBuildResult:
|
||||
images_built: List[str]
|
||||
success: bool
|
||||
error_message: Optional[str] = None
|
||||
class DockerComposeBuild(AbstractStep[List[str]]):
|
||||
def __init__(self,
|
||||
wd: GitClone,
|
||||
docker_repo_username:str,
|
||||
docker_repo_password:str,
|
||||
docker_compose_path:str,
|
||||
docker_repo_url: str,
|
||||
envs: Callable[[], Dict[str, str]], **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self.wd = wd
|
||||
self.envs = envs
|
||||
self.docker_repo_username = docker_repo_username
|
||||
self.docker_repo_password = docker_repo_password
|
||||
self.docker_compose_path = docker_compose_path
|
||||
self.docker_repo_url = docker_repo_url
|
||||
|
||||
class DockerComposeBuild(AbstractStep):
|
||||
def progress(self) -> None:
|
||||
def progress(self) -> List[str]:
|
||||
# def process(self, docker_compose_path: str, docker_repo_url: str, docker_repo_username: str, docker_repo_password: str) -> DockerBuildResult:
|
||||
"""
|
||||
Build and push Docker images defined in a docker-compose file.
|
||||
@@ -25,11 +36,13 @@ class DockerComposeBuild(AbstractStep):
|
||||
try:
|
||||
# Authenticate to docker repo
|
||||
client = docker.from_env()
|
||||
client.login(username=docker_repo_username, password=docker_repo_password, registry=docker_repo_url)
|
||||
|
||||
client.login(username=self.docker_repo_username, password=self.docker_repo_password, registry=self.docker_repo_url)
|
||||
env = self.envs()
|
||||
# Parse docker-compose file
|
||||
with open(docker_compose_path, 'r') as f:
|
||||
compose = yaml.safe_load(f)
|
||||
with open(self.docker_compose_path, 'r') as f:
|
||||
content = f.read()
|
||||
content = re.sub(r'\$\{([^}]+)\}', lambda m: env[m.group(1)], content)
|
||||
compose = yaml.safe_load(content)
|
||||
services = compose.get('services', {})
|
||||
|
||||
for name, svc in services.items():
|
||||
@@ -52,49 +65,49 @@ class DockerComposeBuild(AbstractStep):
|
||||
for line in client.images.push(image, stream=True, decode=True):
|
||||
logger.debug(line)
|
||||
images_built.append(image)
|
||||
return DockerBuildResult(images_built=images_built, success=True)
|
||||
return images_built
|
||||
except Exception as e:
|
||||
logger.error(f"DockerBuild failed: {e}")
|
||||
return DockerBuildResult(images_built=images_built, success=False, error_message=str(e))
|
||||
raise e
|
||||
|
||||
class DockerComposeDeploy(AbstractStep):
|
||||
def progress(self) -> None:
|
||||
# def process(self, docker_compose_path: str, docker_repo_url: str, docker_repo_username: str, docker_repo_password: str) -> DockerBuildResult:
|
||||
class DockerSwarmDeploy(AbstractStep):
|
||||
def __init__(self,
|
||||
wd: GitClone,
|
||||
docker_compose_path: str,
|
||||
envs: Callable[[], Dict[str, str]],
|
||||
stack_name: str,
|
||||
**kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self.wd = wd
|
||||
self.envs = envs
|
||||
self.docker_compose_path = docker_compose_path
|
||||
self.stack_name = stack_name
|
||||
|
||||
def progress(self) -> None:
|
||||
"""
|
||||
Build and push Docker images defined in a docker-compose file.
|
||||
Deploys to Docker Swarm using the specified docker-compose.yaml.
|
||||
"""
|
||||
images_built = []
|
||||
try:
|
||||
# Authenticate to docker repo
|
||||
client = docker.from_env()
|
||||
client.login(username=docker_repo_username, password=docker_repo_password, registry=docker_repo_url)
|
||||
env = self.envs()
|
||||
# Prepare docker-compose file with env substitution
|
||||
with open(self.docker_compose_path, 'r') as f:
|
||||
content = f.read()
|
||||
content = re.sub(r'\$\{([^}]+)\}', lambda m: env.get(m.group(1), ""), content)
|
||||
tmp_compose_path = self.docker_compose_path + ".tmp"
|
||||
with open(tmp_compose_path, 'w') as f:
|
||||
f.write(content)
|
||||
|
||||
# Parse docker-compose file
|
||||
with open(docker_compose_path, 'r') as f:
|
||||
compose = yaml.safe_load(f)
|
||||
services = compose.get('services', {})
|
||||
|
||||
for name, svc in services.items():
|
||||
build_ctx = svc.get('build')
|
||||
image = svc.get('image')
|
||||
if not build_ctx or not image:
|
||||
continue
|
||||
# Check if image exists in remote repo
|
||||
try:
|
||||
client.images.pull(image)
|
||||
logger.info(f"Image {image} already exists in repo, skipping build.")
|
||||
continue
|
||||
except Exception:
|
||||
pass
|
||||
# Build image
|
||||
logger.info(f"Building image {image} from {build_ctx}")
|
||||
client.images.build(path=build_ctx, tag=image)
|
||||
# Push image
|
||||
logger.info(f"Pushing image {image}")
|
||||
for line in client.images.push(image, stream=True, decode=True):
|
||||
logger.debug(line)
|
||||
images_built.append(image)
|
||||
return DockerBuildResult(images_built=images_built, success=True)
|
||||
# Deploy stack to Docker Swarm
|
||||
cmd = [
|
||||
"docker", "stack", "deploy",
|
||||
"-c", tmp_compose_path,
|
||||
self.stack_name
|
||||
]
|
||||
logger.info(f"Deploying stack '{self.stack_name}' using {tmp_compose_path}")
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if result.returncode != 0:
|
||||
logger.error(f"Stack deploy failed: {result.stderr}")
|
||||
raise RuntimeError(f"Stack deploy failed: {result.stderr}")
|
||||
logger.info(f"Stack deployed successfully: {result.stdout}")
|
||||
except Exception as e:
|
||||
logger.error(f"DockerBuild failed: {e}")
|
||||
return DockerBuildResult(images_built=images_built, success=False, error_message=str(e))
|
||||
logger.error(f"DockerSwarmDeploy failed: {e}")
|
||||
raise
|
||||
|
||||
+25
-24
@@ -9,12 +9,11 @@ from enironment import Environment
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@dataclass
|
||||
class GitClone(AbstractStep):
|
||||
env: Environment = field(metadata={"ignore_json":True})
|
||||
result: Union[str, BaseException] = BaseException("No result yet")
|
||||
class GitClone(AbstractStep[str]):
|
||||
def __init__(self, env: Environment, **kwargs):
|
||||
super().__init__(env, **kwargs)
|
||||
|
||||
def progress(self) -> None:
|
||||
def progress(self) -> str | BaseException:
|
||||
try:
|
||||
# Create temporary directory for the clone
|
||||
|
||||
@@ -24,29 +23,34 @@ class GitClone(AbstractStep):
|
||||
logger.info(f"Cloning repository {self.env.repo} to {temp_dir}")
|
||||
git.Repo.clone_from(self.env.repo, temp_dir)
|
||||
|
||||
self.result = temp_dir
|
||||
return temp_dir
|
||||
|
||||
except Exception as e:
|
||||
self.result = e
|
||||
return e
|
||||
|
||||
import hashlib
|
||||
|
||||
@dataclass
|
||||
class CheckoutMerged(AbstractStep):
|
||||
class CheckoutMerged(AbstractStep[Tuple[str, str]]):
|
||||
wd: GitClone
|
||||
branches: List[str]
|
||||
result: Union[Tuple[str, str], BaseException] = BaseException("No result yet")
|
||||
|
||||
def progress(self) -> None:
|
||||
if self.wd.result is BaseException:
|
||||
self.result = self.wd.result
|
||||
return
|
||||
def __init__(self, wd: GitClone, branches: List[str], **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self.wd = wd
|
||||
self.branches = branches
|
||||
|
||||
repo_path = self.wd.result
|
||||
|
||||
def progress(self) -> Tuple[str, str] | BaseException:
|
||||
if self.wd.result_obj is BaseException:
|
||||
return self.wd.result_obj
|
||||
|
||||
repo_path = self.wd.result_obj
|
||||
if not isinstance(repo_path, str):
|
||||
self.result = repo_path # propagate the error (likely an exception)
|
||||
return
|
||||
return BaseException(f"Unknown repo path {repo_path}")
|
||||
|
||||
if len(self.branches) == 0:
|
||||
return BaseException(f"Empty branches set")
|
||||
|
||||
try:
|
||||
# Open the repository
|
||||
repo = git.Repo(repo_path)
|
||||
@@ -66,8 +70,7 @@ class CheckoutMerged(AbstractStep):
|
||||
repo.git.checkout(auto_branch_name)
|
||||
commit_hash = repo.head.commit.hexsha
|
||||
|
||||
self.result = (auto_branch_name, commit_hash)
|
||||
return
|
||||
return (auto_branch_name, commit_hash)
|
||||
|
||||
# Auto branch doesn't exist, create it
|
||||
logger.info(f"Creating new auto branch: {auto_branch_name}")
|
||||
@@ -92,8 +95,7 @@ class CheckoutMerged(AbstractStep):
|
||||
error_message = f"Merge conflict when merging {branch}: {str(e)}"
|
||||
logger.error(error_message)
|
||||
|
||||
self.result = BaseException(error_message)
|
||||
return
|
||||
return BaseException(error_message)
|
||||
|
||||
# Create and push the auto branch
|
||||
repo.git.branch('-f', auto_branch_name)
|
||||
@@ -101,10 +103,9 @@ class CheckoutMerged(AbstractStep):
|
||||
commit_hash = repo.head.commit.hexsha
|
||||
repo.git.push('-f', 'origin', auto_branch_name)
|
||||
|
||||
self.result = (auto_branch_name,commit_hash)
|
||||
return (auto_branch_name,commit_hash)
|
||||
|
||||
except Exception as e:
|
||||
error_message = f"Failed to create merged branch: {str(e)}"
|
||||
logger.error(error_message)
|
||||
self.result = BaseException(error_message)
|
||||
return
|
||||
return BaseException(error_message)
|
||||
|
||||
+37
-2
@@ -1,7 +1,42 @@
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import List, Optional, Union, Tuple, Any
|
||||
from typing import TypeVar, Generic
|
||||
from dataclasses import dataclass, asdict, field
|
||||
from enironment import Environment
|
||||
import logging
|
||||
|
||||
class AbstractStep(ABC):
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
T = TypeVar('T')
|
||||
class AbstractStep(ABC, Generic[T]):
|
||||
|
||||
env: Environment
|
||||
name: str
|
||||
_result: T | BaseException = BaseException("No result yet")
|
||||
|
||||
def __init__(self, env: Environment, n: str | None = None) -> None:
|
||||
if n is None:
|
||||
n = self.__class__.__name__
|
||||
self.name = n
|
||||
self.env = env
|
||||
|
||||
@property
|
||||
def result_obj(self) -> Union[T, BaseException]:
|
||||
return self._result
|
||||
|
||||
@property
|
||||
def result(self) -> T:
|
||||
if isinstance(self._result, BaseException):
|
||||
raise self._result
|
||||
return self._result
|
||||
|
||||
def do_job(self):
|
||||
try:
|
||||
self._result = self.progress()
|
||||
except Exception as e:
|
||||
logger.error(f"Job failed: {e}")
|
||||
self._result = e
|
||||
|
||||
@abstractmethod
|
||||
def progress(self):
|
||||
def progress(self) -> T | BaseException:
|
||||
pass
|
||||
|
||||
+1
-1
@@ -2,7 +2,7 @@ version: '3.8'
|
||||
services:
|
||||
brencher-backend:
|
||||
build: .
|
||||
image: registry.rudolf.keenetic.link/brencher:latest
|
||||
image: registry.rudolf.keenetic.link/brencher:${version}
|
||||
ports:
|
||||
- "5000:5000"
|
||||
environment:
|
||||
|
||||
+13
-18
@@ -3,30 +3,25 @@
|
||||
## Branch List
|
||||
- Display list of branches in all git repositories
|
||||
- Each branch has a checkbox (selected or unselected)
|
||||
- Branch checkbox should be checked if this branch present in EnvironmentDto
|
||||
|
||||
## Branch Refresh
|
||||
- Branches are fetched from server on startup and refreshed every 5 minutes
|
||||
|
||||
## Communication
|
||||
|
||||
## Environment Editing
|
||||
- EnvironmentDto properties editable by user:
|
||||
branches
|
||||
state
|
||||
|
||||
## Environment State
|
||||
- On each update for EnvironmentDto render pipeline on right conlumn on UI
|
||||
|
||||
## Communication and errors
|
||||
- UI communicates with backend using websocket with class-based mapping
|
||||
- UI receives environment updates via websocket
|
||||
- UI receives errors via websocket to display
|
||||
|
||||
## Error Handling
|
||||
- If request fails, show error message in bar
|
||||
- If backend response is not 2xx, display error with message from response
|
||||
|
||||
## Environment Editing
|
||||
- Environment is editable by user
|
||||
- EnvironmentDto contains list of branches
|
||||
|
||||
## Environment State
|
||||
- Display list of jobs from EnvironmentState
|
||||
|
||||
## DTOs
|
||||
- EnvironmentDto: { branches: [string] }
|
||||
- EnvironmentStateDto: { jobs: [JobDto] }
|
||||
|
||||
## Notes
|
||||
- UI should handle websocket events for environment updates and errors
|
||||
- UI should provide feedback for failed requests and backend errors
|
||||
- If request fails, show error message in bar
|
||||
- If backend response is not 2xx, display error with message from response
|
||||
|
||||
+9
-5
@@ -24,6 +24,7 @@ let defaultState = 'Active';
|
||||
let environment = null;
|
||||
let jobs = [];
|
||||
let branchStates = {};
|
||||
|
||||
let wsBranches = null;
|
||||
let wsEnv = null;
|
||||
let wsErr = null;
|
||||
@@ -88,15 +89,17 @@ function renderReleases() {
|
||||
function renderJobs() {
|
||||
const jobsList = document.getElementById('jobs-list');
|
||||
if (!jobsList) return;
|
||||
if (!jobs.length) {
|
||||
if (!environment) return;
|
||||
if (!environment.length) {
|
||||
jobsList.innerHTML = '<p class="loading">No jobs found.</p>';
|
||||
return;
|
||||
}
|
||||
jobsList.innerHTML = jobs.map(job => `
|
||||
<div class="job-item">
|
||||
|
||||
jobsList.innerHTML = environment[0][1].map(job => {
|
||||
return `<div class="job-item">
|
||||
<strong>${job.name}</strong> - ${job.status}
|
||||
</div>
|
||||
`).join('');
|
||||
</div>`
|
||||
}).join('');
|
||||
}
|
||||
|
||||
stateSelector.onchange = (e) => {
|
||||
@@ -123,6 +126,7 @@ function setupSocketIO() {
|
||||
wsEnv.on('environments', (data) => {
|
||||
environment = data;
|
||||
renderBranches();
|
||||
renderJobs()
|
||||
showStatus('Environment updated via Socket.IO.');
|
||||
});
|
||||
|
||||
|
||||
Reference in New Issue
Block a user