Pipeline started

This commit is contained in:
2025-08-07 01:10:58 +02:00
parent 067d86032b
commit eb09da9f30
6 changed files with 92 additions and 253 deletions
+19 -14
View File
@@ -35,7 +35,7 @@ state_lock = threading.Lock()
def fetch_branches():
global branches, environments
for e in environments:
for e, _ in environments:
url = e.repo
protocol, rest = url.split('://')
# Get environment variables
@@ -58,7 +58,6 @@ def fetch_branches():
new_branches.append(branch_name)
branches[e.id] = new_branches
socketio.emit('branches_updated', {'branches': branches})
logger.info(f"Fetched {e.id}:{len(new_branches)} branches")
except Exception as e:
@@ -75,38 +74,32 @@ def serve_index():
def serve_static(path):
return send_from_directory(FRONTEND_DIR, path)
# Socket.IO events
@socketio.on('connect')
def on_connect():
emit('branches_updated', {'branches': branches})
# emit('releases_updated', {'releases': [asdict(r) for r in releases.values()]})
# --- WebSocket Endpoints ---
from flask import copy_current_request_context
from flask_socketio import Namespace
class BranchesNamespace(Namespace):
def on_connect(self):
def on_connect(self, auth=None):
emit('branches', branches)
def on_update(self, data):
emit('branches', branches)
class EnvironmentNamespace(Namespace):
def on_connect(self):
env_dtos = [asdict(e) for e in environments]
def on_connect(self, auth=None):
env_dtos = [asdict(e) for e, _ in environments]
emit('environments', env_dtos)
def on_update(self, data):
# Accept environment update from UI
# Update environments and broadcast
for e in environments:
for e, _ in environments:
if e.id == data.get('id'):
e.state = data.get('state', e.state)
e.branches = data.get('branches', e.branches)
env_dtos = [asdict(e) for e in environments]
env_dtos = [asdict(e) for e, _ in environments]
emit('environments', env_dtos, broadcast=True)
class ErrorsNamespace(Namespace):
def on_connect(self):
def on_connect(self, auth=None):
pass
def on_error(self, data):
emit('error', data)
@@ -120,6 +113,7 @@ if __name__ == '__main__':
def branch_refresh_thread():
while True:
fetch_branches()
socketio.emit('branches', branches)
time.sleep(300) # 5 minutes
# Start branch refresh thread
@@ -127,5 +121,16 @@ if __name__ == '__main__':
refresh_thread.daemon = True
refresh_thread.start()
def processing_thread():
while True:
import processing
processing.do_job(environments, lambda e: socketio.emit('error', e))
env_dtos = [asdict(e) for e, _ in environments]
socketio.emit('environments', env_dtos)
time.sleep(10)
processing = threading.Thread(target=processing_thread)
processing.daemon = True
processing.start()
# Run the server
socketio.run(app, host='0.0.0.0', port=5000, debug=True)
+9 -8
View File
@@ -3,20 +3,20 @@ from steps.git import CheckoutMerged
from steps.docker import DockerComposeBuild
from steps.docker import DockerComposeDeploy
from enironment import Environment
from typing import List, Dict, Any, Optional
from typing import List, Dict, Any, Optional, Tuple
from steps.step import AbstractStep
brencher = Environment(
env = Environment(
id="brencher",
branches=[],
state="Pause",
branches=["main"],
state="Active",
repo="https://github.com/rudolf1/brencher.git",
pipeline=lambda e: pipeline(e)
)
def pipeline(env: Environment) -> List[AbstractStep]:
clone = GitClone()
checkoutMerged = CheckoutMerged() #wd=lambda: clone.repo, env.branches)
def create_pipeline(env: Environment) -> List[AbstractStep]:
clone = GitClone(env)
checkoutMerged = CheckoutMerged(clone, env.branches)
buildDocker = DockerComposeBuild() #wd=lambda: clone.path, version=lambda: checkoutMerged.version)
deployDocker = DockerComposeDeploy() #version=lambda: checkoutMerged.version)
return [
@@ -26,4 +26,5 @@ def pipeline(env: Environment) -> List[AbstractStep]:
deployDocker
]
brencher: Tuple[Environment, List[AbstractStep]] = (env, create_pipeline(env))
+2 -2
View File
@@ -5,7 +5,7 @@ from steps.step import AbstractStep
@dataclass
class Environment:
id: str
branches: List[Tuple[str, str]]
branches: List[str]
state: str # 'Active' or 'Pause'
repo: str # git repo
pipeline: Callable[[Any], List[AbstractStep]]
# pipeline: List[AbstractStep]
+17 -125
View File
@@ -1,130 +1,22 @@
# Import step modules
from backend.steps.git import GitClone, GitCloneResult
from backend.steps.checkout_merged import CheckoutMergedResult
from backend.steps.checkout_merged import CheckoutMerged
from steps.gradle_build import GradleBuild, GradleBuildResult
from enironment import Environment
from steps.step import AbstractStep
from typing import List, Dict, Any, Optional, Tuple, Callable
import logging
# Process releases flow
def process_release(release_id):
if release_id not in releases:
return
release = releases[release_id]
if release.state != 'Active':
return
try:
# Step 1: Git Clone
git_clone = GitClone()
clone_result = git_clone.process(release.git_url)
with state_lock:
release.clone = clone_result
socketio.emit('release_updated', asdict(release))
if not clone_result.success:
logger.error(f"Git clone failed for release {release_id}: {clone_result.error_message}")
return
# Step 2: Checkout Merged
checkout_merged = CheckoutMerged()
checkout_result = checkout_merged.process(clone_result, release.branches)
with state_lock:
release.checkout_merged = checkout_result
socketio.emit('release_updated', asdict(release))
if not checkout_result.success:
logger.error(f"Checkout merged failed for release {release_id}: {checkout_result.error_message}")
return
# Step 3: Gradle Build
gradle_build = GradleBuild()
build_result = gradle_build.process(checkout_result, "jib")
with state_lock:
release.gradle_build = build_result
socketio.emit('release_updated', asdict(release))
if not build_result.success:
logger.error(f"Gradle build failed for release {release_id}: {build_result.error_message}")
return
except Exception as e:
error_msg = f"Error processing release {release_id}: {str(e)}"
logger.error(error_msg)
socketio.emit('error', {'message': error_msg})
logger = logging.getLogger(__name__)
@app.route('/api/releases', methods=['GET'])
def get_releases():
return jsonify([asdict(r) for r in releases.values()])
@app.route('/api/releases', methods=['POST'])
def create_release():
data = request.json
branches_to_merge = data.get('branches', [])
state = data.get('state', 'Pause')
def do_job(environemnts: List[Tuple[Environment, List[AbstractStep]]], onerror: Callable[[str], None]):
for env, pipe in environemnts:
try:
if env.state != 'Active':
continue
for step in pipe:
if isinstance(step, AbstractStep):
step.progress()
if not branches_to_merge:
return jsonify({'error': 'No branches selected'}), 400
release_id = hashlib.md5(''.join(sorted(branches_to_merge)).encode()).hexdigest()
if release_id in releases:
return jsonify({'error': 'Release with these branches already exists'}), 409
release = ReleaseDto(
id=release_id,
branches=branches_to_merge,
state=state,
git_url=GIT_REPO_URL
)
with state_lock:
releases[release_id] = release
socketio.emit('release_created', asdict(release))
if state == 'Active':
thread = threading.Thread(target=process_release, args=(release_id,))
thread.daemon = True
thread.start()
return jsonify(asdict(release)), 201
@app.route('/api/releases/<release_id>', methods=['PUT'])
def update_release(release_id):
if release_id not in releases:
return jsonify({'error': 'Release not found'}), 404
data = request.json
release = releases[release_id]
old_state = release.state
new_state = data.get('state', old_state)
if old_state != new_state:
release.state = new_state
with state_lock:
releases[release_id] = release
socketio.emit('release_updated', asdict(release))
if old_state == 'Pause' and new_state == 'Active':
thread = threading.Thread(target=process_release, args=(release_id,))
thread.daemon = True
thread.start()
return jsonify(asdict(release))
except Exception as e:
error_msg = f"Error processing release {env.id}: {str(e)}"
logger.error(error_msg)
onerror(error_msg)
@app.route('/api/releases/<release_id>', methods=['DELETE'])
def delete_release(release_id):
if release_id not in releases:
return jsonify({'error': 'Release not found'}), 404
with state_lock:
release = releases.pop(release_id)
socketio.emit('release_deleted', {'id': release_id})
return jsonify({'success': True})
+36 -77
View File
@@ -2,86 +2,57 @@ import tempfile
import git
import logging
from dataclasses import dataclass, asdict, field
from typing import List, Optional
from typing import List, Optional, Union, Tuple
from steps.step import AbstractStep
import tempfile
from enironment import Environment
logger = logging.getLogger(__name__)
@dataclass
class GitCloneResult:
repo_path: str
success: bool
error_message: Optional[str] = None
class GitClone(AbstractStep):
env: Environment = field(metadata={"ignore_json":True})
result: Union[str, BaseException] = BaseException("No result yet")
def progress(self) -> None:
"""
Clone the Git repository to a temporary folder.
Args:
git_url: URL of the Git repository to clone
Returns:
GitCloneResult: Object containing the path to the cloned repo and status
"""
try:
# Create temporary directory for the clone
temp_dir = tempfile.mkdtemp(prefix="brencher_")
temp_dir = tempfile.mkdtemp(prefix=f"{self.env.id}_")
# Clone the repository
logger.info(f"Cloning repository {git_url} to {temp_dir}")
git.Repo.clone_from(git_url, temp_dir)
logger.info(f"Cloning repository {self.env.repo} to {temp_dir}")
git.Repo.clone_from(self.env.repo, temp_dir)
return GitCloneResult(
repo_path=temp_dir,
success=True
)
self.result = temp_dir
except Exception as e:
error_message = f"Failed to clone repository: {str(e)}"
logger.error(error_message)
return GitCloneResult(
repo_path="",
success=False,
error_message=error_message
)
self.result = e
import hashlib
@dataclass
class CheckoutMergedResult:
branch_name: Optional[str] = None
commit_hash: Optional[str] = None
success: bool = True
error_message: Optional[str] = None
class CheckoutMerged(AbstractStep):
wd: GitClone
branches: List[str]
result: Union[Tuple[str, str], BaseException] = BaseException("No result yet")
def progress(self) -> None:
"""
Find or create an auto branch that represents the merged state of selected branches.
Args:
clone_result: Result from GitClone step containing repo path
branches: List of branches to merge
Returns:
CheckoutMergedResult: Object with merged branch details
"""
if not clone_result.success:
return CheckoutMergedResult(
success=False,
error_message="Cannot checkout merged branches: Git clone failed"
)
repo_path = clone_result.repo_path
if self.wd.result is BaseException:
self.result = self.wd.result
return
repo_path = self.wd.result
if not isinstance(repo_path, str):
self.result = repo_path # propagate the error (likely an exception)
return
try:
# Open the repository
repo = git.Repo(repo_path)
# Generate hash from branch names
sorted_branches = sorted(branches)
sorted_branches = sorted(self.branches)
branch_hash = hashlib.sha1(''.join(sorted_branches).encode()).hexdigest()
auto_branch_name = f"auto/{branch_hash}"
@@ -95,17 +66,14 @@ class CheckoutMerged(AbstractStep):
repo.git.checkout(auto_branch_name)
commit_hash = repo.head.commit.hexsha
return CheckoutMergedResult(
branch_name=auto_branch_name,
commit_hash=commit_hash,
success=True
)
self.result = (auto_branch_name, commit_hash)
return
# Auto branch doesn't exist, create it
logger.info(f"Creating new auto branch: {auto_branch_name}")
# Start with the first branch
base_branch = branches[0]
base_branch = self.branches[0]
repo.git.checkout(base_branch)
# Create temporary branch for merging
@@ -113,7 +81,7 @@ class CheckoutMerged(AbstractStep):
repo.git.checkout('-b', temp_branch)
# Merge the rest of the branches
for branch in branches[1:]:
for branch in self.branches[1:]:
try:
logger.info(f"Merging branch: {branch}")
repo.git.merge(branch, '--no-ff')
@@ -124,10 +92,8 @@ class CheckoutMerged(AbstractStep):
error_message = f"Merge conflict when merging {branch}: {str(e)}"
logger.error(error_message)
return CheckoutMergedResult(
success=False,
error_message=error_message
)
self.result = BaseException(error_message)
return
# Create and push the auto branch
repo.git.branch('-f', auto_branch_name)
@@ -135,17 +101,10 @@ class CheckoutMerged(AbstractStep):
commit_hash = repo.head.commit.hexsha
repo.git.push('-f', 'origin', auto_branch_name)
return CheckoutMergedResult(
branch_name=auto_branch_name,
commit_hash=commit_hash,
success=True
)
self.result = (auto_branch_name,commit_hash)
except Exception as e:
error_message = f"Failed to create merged branch: {str(e)}"
logger.error(error_message)
return CheckoutMergedResult(
success=False,
error_message=error_message
)
self.result = BaseException(error_message)
return
+9 -27
View File
@@ -4,7 +4,6 @@ console.log('app.js loaded');
const socket = io();
const ENVIRONMENT_API = '/environment';
const WS_BRANCHES = '/ws/branches';
const WS_ENVIRONMENT = '/ws/environment';
const WS_ERRORS = '/ws/errors';
@@ -37,27 +36,6 @@ function showStatus(message, isError = false) {
}
closeStatus.onclick = () => statusBar.classList.add('hidden');
function fetchEnvironment() {
fetch(ENVIRONMENT_API)
.then(res => res.ok ? res.json() : Promise.reject(res))
.then(data => {
environment = data;
branchStates = {};
if (Array.isArray(environment)) {
environment.forEach(env => {
env.branches.forEach(([envName, branchName]) => {
branchStates[branchName] = env.state;
});
});
}
renderBranches();
renderJobs();
})
.catch(err => {
showStatus('Failed to load environment', true);
});
}
function renderBranches() {
if (!branches.length) {
branchesList.innerHTML = '<p class="loading">No branches found.</p>';
@@ -132,25 +110,29 @@ refreshBranchesBtn.onclick = () => {
// Socket.IO setup
function setupSocketIO() {
socket.on('branches_updated', (data) => {
branches = Object.entries(data.branches).flatMap(([env, branchList]) => branchList.map(branch => ({ env, branch })));
wsBranches = io(WS_BRANCHES);
wsEnv = io(WS_ENVIRONMENT);
wsErr = io(WS_ERRORS);
wsBranches.on('branches', (data) => {
branches = Object.entries(data).flatMap(([env, branchList]) => branchList.map(branch => ({ env, branch })));
renderBranches();
showStatus('Branches updated via Socket.IO.');
});
socket.on('environments', (data) => {
wsEnv.on('environments', (data) => {
environment = data;
renderBranches();
showStatus('Environment updated via Socket.IO.');
});
socket.on('error', (data) => {
wsErr.on('error', (data) => {
showStatus(data.message || 'Unknown error', true);
});
}
function sendEnvironmentUpdate(envUpdate) {
socket.emit('update', envUpdate);
wsEnv.emit('update', envUpdate);
}
function updateEnvironment() {