forked from admin/brencher
Pipeline started
This commit is contained in:
+19
-14
@@ -35,7 +35,7 @@ state_lock = threading.Lock()
|
||||
def fetch_branches():
|
||||
global branches, environments
|
||||
|
||||
for e in environments:
|
||||
for e, _ in environments:
|
||||
url = e.repo
|
||||
protocol, rest = url.split('://')
|
||||
# Get environment variables
|
||||
@@ -58,7 +58,6 @@ def fetch_branches():
|
||||
new_branches.append(branch_name)
|
||||
|
||||
branches[e.id] = new_branches
|
||||
socketio.emit('branches_updated', {'branches': branches})
|
||||
logger.info(f"Fetched {e.id}:{len(new_branches)} branches")
|
||||
|
||||
except Exception as e:
|
||||
@@ -75,38 +74,32 @@ def serve_index():
|
||||
def serve_static(path):
|
||||
return send_from_directory(FRONTEND_DIR, path)
|
||||
|
||||
# Socket.IO events
|
||||
@socketio.on('connect')
|
||||
def on_connect():
|
||||
emit('branches_updated', {'branches': branches})
|
||||
# emit('releases_updated', {'releases': [asdict(r) for r in releases.values()]})
|
||||
|
||||
# --- WebSocket Endpoints ---
|
||||
from flask import copy_current_request_context
|
||||
from flask_socketio import Namespace
|
||||
|
||||
class BranchesNamespace(Namespace):
|
||||
def on_connect(self):
|
||||
def on_connect(self, auth=None):
|
||||
emit('branches', branches)
|
||||
def on_update(self, data):
|
||||
emit('branches', branches)
|
||||
|
||||
class EnvironmentNamespace(Namespace):
|
||||
def on_connect(self):
|
||||
env_dtos = [asdict(e) for e in environments]
|
||||
def on_connect(self, auth=None):
|
||||
env_dtos = [asdict(e) for e, _ in environments]
|
||||
emit('environments', env_dtos)
|
||||
def on_update(self, data):
|
||||
# Accept environment update from UI
|
||||
# Update environments and broadcast
|
||||
for e in environments:
|
||||
for e, _ in environments:
|
||||
if e.id == data.get('id'):
|
||||
e.state = data.get('state', e.state)
|
||||
e.branches = data.get('branches', e.branches)
|
||||
env_dtos = [asdict(e) for e in environments]
|
||||
env_dtos = [asdict(e) for e, _ in environments]
|
||||
emit('environments', env_dtos, broadcast=True)
|
||||
|
||||
class ErrorsNamespace(Namespace):
|
||||
def on_connect(self):
|
||||
def on_connect(self, auth=None):
|
||||
pass
|
||||
def on_error(self, data):
|
||||
emit('error', data)
|
||||
@@ -120,6 +113,7 @@ if __name__ == '__main__':
|
||||
def branch_refresh_thread():
|
||||
while True:
|
||||
fetch_branches()
|
||||
socketio.emit('branches', branches)
|
||||
time.sleep(300) # 5 minutes
|
||||
|
||||
# Start branch refresh thread
|
||||
@@ -127,5 +121,16 @@ if __name__ == '__main__':
|
||||
refresh_thread.daemon = True
|
||||
refresh_thread.start()
|
||||
|
||||
def processing_thread():
|
||||
while True:
|
||||
import processing
|
||||
processing.do_job(environments, lambda e: socketio.emit('error', e))
|
||||
env_dtos = [asdict(e) for e, _ in environments]
|
||||
socketio.emit('environments', env_dtos)
|
||||
time.sleep(10)
|
||||
processing = threading.Thread(target=processing_thread)
|
||||
processing.daemon = True
|
||||
processing.start()
|
||||
|
||||
# Run the server
|
||||
socketio.run(app, host='0.0.0.0', port=5000, debug=True)
|
||||
|
||||
@@ -3,20 +3,20 @@ from steps.git import CheckoutMerged
|
||||
from steps.docker import DockerComposeBuild
|
||||
from steps.docker import DockerComposeDeploy
|
||||
from enironment import Environment
|
||||
from typing import List, Dict, Any, Optional
|
||||
from typing import List, Dict, Any, Optional, Tuple
|
||||
from steps.step import AbstractStep
|
||||
|
||||
brencher = Environment(
|
||||
env = Environment(
|
||||
id="brencher",
|
||||
branches=[],
|
||||
state="Pause",
|
||||
branches=["main"],
|
||||
state="Active",
|
||||
repo="https://github.com/rudolf1/brencher.git",
|
||||
pipeline=lambda e: pipeline(e)
|
||||
)
|
||||
|
||||
def pipeline(env: Environment) -> List[AbstractStep]:
|
||||
clone = GitClone()
|
||||
checkoutMerged = CheckoutMerged() #wd=lambda: clone.repo, env.branches)
|
||||
|
||||
def create_pipeline(env: Environment) -> List[AbstractStep]:
|
||||
clone = GitClone(env)
|
||||
checkoutMerged = CheckoutMerged(clone, env.branches)
|
||||
buildDocker = DockerComposeBuild() #wd=lambda: clone.path, version=lambda: checkoutMerged.version)
|
||||
deployDocker = DockerComposeDeploy() #version=lambda: checkoutMerged.version)
|
||||
return [
|
||||
@@ -26,4 +26,5 @@ def pipeline(env: Environment) -> List[AbstractStep]:
|
||||
deployDocker
|
||||
]
|
||||
|
||||
brencher: Tuple[Environment, List[AbstractStep]] = (env, create_pipeline(env))
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ from steps.step import AbstractStep
|
||||
@dataclass
|
||||
class Environment:
|
||||
id: str
|
||||
branches: List[Tuple[str, str]]
|
||||
branches: List[str]
|
||||
state: str # 'Active' or 'Pause'
|
||||
repo: str # git repo
|
||||
pipeline: Callable[[Any], List[AbstractStep]]
|
||||
# pipeline: List[AbstractStep]
|
||||
|
||||
+17
-125
@@ -1,130 +1,22 @@
|
||||
# Import step modules
|
||||
from backend.steps.git import GitClone, GitCloneResult
|
||||
from backend.steps.checkout_merged import CheckoutMergedResult
|
||||
from backend.steps.checkout_merged import CheckoutMerged
|
||||
from steps.gradle_build import GradleBuild, GradleBuildResult
|
||||
from enironment import Environment
|
||||
from steps.step import AbstractStep
|
||||
from typing import List, Dict, Any, Optional, Tuple, Callable
|
||||
import logging
|
||||
|
||||
# Process releases flow
|
||||
def process_release(release_id):
|
||||
if release_id not in releases:
|
||||
return
|
||||
|
||||
release = releases[release_id]
|
||||
if release.state != 'Active':
|
||||
return
|
||||
|
||||
try:
|
||||
# Step 1: Git Clone
|
||||
git_clone = GitClone()
|
||||
clone_result = git_clone.process(release.git_url)
|
||||
|
||||
with state_lock:
|
||||
release.clone = clone_result
|
||||
socketio.emit('release_updated', asdict(release))
|
||||
|
||||
if not clone_result.success:
|
||||
logger.error(f"Git clone failed for release {release_id}: {clone_result.error_message}")
|
||||
return
|
||||
|
||||
# Step 2: Checkout Merged
|
||||
checkout_merged = CheckoutMerged()
|
||||
checkout_result = checkout_merged.process(clone_result, release.branches)
|
||||
|
||||
with state_lock:
|
||||
release.checkout_merged = checkout_result
|
||||
socketio.emit('release_updated', asdict(release))
|
||||
|
||||
if not checkout_result.success:
|
||||
logger.error(f"Checkout merged failed for release {release_id}: {checkout_result.error_message}")
|
||||
return
|
||||
|
||||
# Step 3: Gradle Build
|
||||
gradle_build = GradleBuild()
|
||||
build_result = gradle_build.process(checkout_result, "jib")
|
||||
|
||||
with state_lock:
|
||||
release.gradle_build = build_result
|
||||
socketio.emit('release_updated', asdict(release))
|
||||
|
||||
if not build_result.success:
|
||||
logger.error(f"Gradle build failed for release {release_id}: {build_result.error_message}")
|
||||
return
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"Error processing release {release_id}: {str(e)}"
|
||||
logger.error(error_msg)
|
||||
socketio.emit('error', {'message': error_msg})
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@app.route('/api/releases', methods=['GET'])
|
||||
def get_releases():
|
||||
return jsonify([asdict(r) for r in releases.values()])
|
||||
|
||||
@app.route('/api/releases', methods=['POST'])
|
||||
def create_release():
|
||||
data = request.json
|
||||
branches_to_merge = data.get('branches', [])
|
||||
state = data.get('state', 'Pause')
|
||||
def do_job(environemnts: List[Tuple[Environment, List[AbstractStep]]], onerror: Callable[[str], None]):
|
||||
for env, pipe in environemnts:
|
||||
try:
|
||||
if env.state != 'Active':
|
||||
continue
|
||||
for step in pipe:
|
||||
if isinstance(step, AbstractStep):
|
||||
step.progress()
|
||||
|
||||
if not branches_to_merge:
|
||||
return jsonify({'error': 'No branches selected'}), 400
|
||||
|
||||
release_id = hashlib.md5(''.join(sorted(branches_to_merge)).encode()).hexdigest()
|
||||
|
||||
if release_id in releases:
|
||||
return jsonify({'error': 'Release with these branches already exists'}), 409
|
||||
|
||||
release = ReleaseDto(
|
||||
id=release_id,
|
||||
branches=branches_to_merge,
|
||||
state=state,
|
||||
git_url=GIT_REPO_URL
|
||||
)
|
||||
|
||||
with state_lock:
|
||||
releases[release_id] = release
|
||||
|
||||
socketio.emit('release_created', asdict(release))
|
||||
|
||||
if state == 'Active':
|
||||
thread = threading.Thread(target=process_release, args=(release_id,))
|
||||
thread.daemon = True
|
||||
thread.start()
|
||||
|
||||
return jsonify(asdict(release)), 201
|
||||
|
||||
@app.route('/api/releases/<release_id>', methods=['PUT'])
|
||||
def update_release(release_id):
|
||||
if release_id not in releases:
|
||||
return jsonify({'error': 'Release not found'}), 404
|
||||
|
||||
data = request.json
|
||||
release = releases[release_id]
|
||||
old_state = release.state
|
||||
new_state = data.get('state', old_state)
|
||||
|
||||
if old_state != new_state:
|
||||
release.state = new_state
|
||||
|
||||
with state_lock:
|
||||
releases[release_id] = release
|
||||
|
||||
socketio.emit('release_updated', asdict(release))
|
||||
|
||||
if old_state == 'Pause' and new_state == 'Active':
|
||||
thread = threading.Thread(target=process_release, args=(release_id,))
|
||||
thread.daemon = True
|
||||
thread.start()
|
||||
|
||||
return jsonify(asdict(release))
|
||||
except Exception as e:
|
||||
error_msg = f"Error processing release {env.id}: {str(e)}"
|
||||
logger.error(error_msg)
|
||||
onerror(error_msg)
|
||||
|
||||
@app.route('/api/releases/<release_id>', methods=['DELETE'])
|
||||
def delete_release(release_id):
|
||||
if release_id not in releases:
|
||||
return jsonify({'error': 'Release not found'}), 404
|
||||
|
||||
with state_lock:
|
||||
release = releases.pop(release_id)
|
||||
|
||||
socketio.emit('release_deleted', {'id': release_id})
|
||||
|
||||
return jsonify({'success': True})
|
||||
|
||||
+36
-77
@@ -2,86 +2,57 @@ import tempfile
|
||||
import git
|
||||
import logging
|
||||
from dataclasses import dataclass, asdict, field
|
||||
from typing import List, Optional
|
||||
from typing import List, Optional, Union, Tuple
|
||||
from steps.step import AbstractStep
|
||||
import tempfile
|
||||
from enironment import Environment
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@dataclass
|
||||
class GitCloneResult:
|
||||
repo_path: str
|
||||
success: bool
|
||||
error_message: Optional[str] = None
|
||||
|
||||
class GitClone(AbstractStep):
|
||||
env: Environment = field(metadata={"ignore_json":True})
|
||||
result: Union[str, BaseException] = BaseException("No result yet")
|
||||
|
||||
def progress(self) -> None:
|
||||
"""
|
||||
Clone the Git repository to a temporary folder.
|
||||
|
||||
Args:
|
||||
git_url: URL of the Git repository to clone
|
||||
|
||||
Returns:
|
||||
GitCloneResult: Object containing the path to the cloned repo and status
|
||||
"""
|
||||
try:
|
||||
# Create temporary directory for the clone
|
||||
temp_dir = tempfile.mkdtemp(prefix="brencher_")
|
||||
|
||||
temp_dir = tempfile.mkdtemp(prefix=f"{self.env.id}_")
|
||||
|
||||
# Clone the repository
|
||||
logger.info(f"Cloning repository {git_url} to {temp_dir}")
|
||||
git.Repo.clone_from(git_url, temp_dir)
|
||||
logger.info(f"Cloning repository {self.env.repo} to {temp_dir}")
|
||||
git.Repo.clone_from(self.env.repo, temp_dir)
|
||||
|
||||
return GitCloneResult(
|
||||
repo_path=temp_dir,
|
||||
success=True
|
||||
)
|
||||
self.result = temp_dir
|
||||
|
||||
except Exception as e:
|
||||
error_message = f"Failed to clone repository: {str(e)}"
|
||||
logger.error(error_message)
|
||||
|
||||
return GitCloneResult(
|
||||
repo_path="",
|
||||
success=False,
|
||||
error_message=error_message
|
||||
)
|
||||
|
||||
self.result = e
|
||||
|
||||
import hashlib
|
||||
|
||||
@dataclass
|
||||
class CheckoutMergedResult:
|
||||
branch_name: Optional[str] = None
|
||||
commit_hash: Optional[str] = None
|
||||
success: bool = True
|
||||
error_message: Optional[str] = None
|
||||
|
||||
class CheckoutMerged(AbstractStep):
|
||||
wd: GitClone
|
||||
branches: List[str]
|
||||
result: Union[Tuple[str, str], BaseException] = BaseException("No result yet")
|
||||
|
||||
def progress(self) -> None:
|
||||
"""
|
||||
Find or create an auto branch that represents the merged state of selected branches.
|
||||
|
||||
Args:
|
||||
clone_result: Result from GitClone step containing repo path
|
||||
branches: List of branches to merge
|
||||
|
||||
Returns:
|
||||
CheckoutMergedResult: Object with merged branch details
|
||||
"""
|
||||
if not clone_result.success:
|
||||
return CheckoutMergedResult(
|
||||
success=False,
|
||||
error_message="Cannot checkout merged branches: Git clone failed"
|
||||
)
|
||||
|
||||
repo_path = clone_result.repo_path
|
||||
if self.wd.result is BaseException:
|
||||
self.result = self.wd.result
|
||||
return
|
||||
|
||||
repo_path = self.wd.result
|
||||
if not isinstance(repo_path, str):
|
||||
self.result = repo_path # propagate the error (likely an exception)
|
||||
return
|
||||
|
||||
try:
|
||||
# Open the repository
|
||||
repo = git.Repo(repo_path)
|
||||
|
||||
# Generate hash from branch names
|
||||
sorted_branches = sorted(branches)
|
||||
sorted_branches = sorted(self.branches)
|
||||
branch_hash = hashlib.sha1(''.join(sorted_branches).encode()).hexdigest()
|
||||
auto_branch_name = f"auto/{branch_hash}"
|
||||
|
||||
@@ -95,17 +66,14 @@ class CheckoutMerged(AbstractStep):
|
||||
repo.git.checkout(auto_branch_name)
|
||||
commit_hash = repo.head.commit.hexsha
|
||||
|
||||
return CheckoutMergedResult(
|
||||
branch_name=auto_branch_name,
|
||||
commit_hash=commit_hash,
|
||||
success=True
|
||||
)
|
||||
|
||||
self.result = (auto_branch_name, commit_hash)
|
||||
return
|
||||
|
||||
# Auto branch doesn't exist, create it
|
||||
logger.info(f"Creating new auto branch: {auto_branch_name}")
|
||||
|
||||
# Start with the first branch
|
||||
base_branch = branches[0]
|
||||
base_branch = self.branches[0]
|
||||
repo.git.checkout(base_branch)
|
||||
|
||||
# Create temporary branch for merging
|
||||
@@ -113,7 +81,7 @@ class CheckoutMerged(AbstractStep):
|
||||
repo.git.checkout('-b', temp_branch)
|
||||
|
||||
# Merge the rest of the branches
|
||||
for branch in branches[1:]:
|
||||
for branch in self.branches[1:]:
|
||||
try:
|
||||
logger.info(f"Merging branch: {branch}")
|
||||
repo.git.merge(branch, '--no-ff')
|
||||
@@ -124,10 +92,8 @@ class CheckoutMerged(AbstractStep):
|
||||
error_message = f"Merge conflict when merging {branch}: {str(e)}"
|
||||
logger.error(error_message)
|
||||
|
||||
return CheckoutMergedResult(
|
||||
success=False,
|
||||
error_message=error_message
|
||||
)
|
||||
self.result = BaseException(error_message)
|
||||
return
|
||||
|
||||
# Create and push the auto branch
|
||||
repo.git.branch('-f', auto_branch_name)
|
||||
@@ -135,17 +101,10 @@ class CheckoutMerged(AbstractStep):
|
||||
commit_hash = repo.head.commit.hexsha
|
||||
repo.git.push('-f', 'origin', auto_branch_name)
|
||||
|
||||
return CheckoutMergedResult(
|
||||
branch_name=auto_branch_name,
|
||||
commit_hash=commit_hash,
|
||||
success=True
|
||||
)
|
||||
self.result = (auto_branch_name,commit_hash)
|
||||
|
||||
except Exception as e:
|
||||
error_message = f"Failed to create merged branch: {str(e)}"
|
||||
logger.error(error_message)
|
||||
|
||||
return CheckoutMergedResult(
|
||||
success=False,
|
||||
error_message=error_message
|
||||
)
|
||||
self.result = BaseException(error_message)
|
||||
return
|
||||
|
||||
+9
-27
@@ -4,7 +4,6 @@ console.log('app.js loaded');
|
||||
|
||||
const socket = io();
|
||||
|
||||
const ENVIRONMENT_API = '/environment';
|
||||
const WS_BRANCHES = '/ws/branches';
|
||||
const WS_ENVIRONMENT = '/ws/environment';
|
||||
const WS_ERRORS = '/ws/errors';
|
||||
@@ -37,27 +36,6 @@ function showStatus(message, isError = false) {
|
||||
}
|
||||
closeStatus.onclick = () => statusBar.classList.add('hidden');
|
||||
|
||||
function fetchEnvironment() {
|
||||
fetch(ENVIRONMENT_API)
|
||||
.then(res => res.ok ? res.json() : Promise.reject(res))
|
||||
.then(data => {
|
||||
environment = data;
|
||||
branchStates = {};
|
||||
if (Array.isArray(environment)) {
|
||||
environment.forEach(env => {
|
||||
env.branches.forEach(([envName, branchName]) => {
|
||||
branchStates[branchName] = env.state;
|
||||
});
|
||||
});
|
||||
}
|
||||
renderBranches();
|
||||
renderJobs();
|
||||
})
|
||||
.catch(err => {
|
||||
showStatus('Failed to load environment', true);
|
||||
});
|
||||
}
|
||||
|
||||
function renderBranches() {
|
||||
if (!branches.length) {
|
||||
branchesList.innerHTML = '<p class="loading">No branches found.</p>';
|
||||
@@ -132,25 +110,29 @@ refreshBranchesBtn.onclick = () => {
|
||||
|
||||
// Socket.IO setup
|
||||
function setupSocketIO() {
|
||||
socket.on('branches_updated', (data) => {
|
||||
branches = Object.entries(data.branches).flatMap(([env, branchList]) => branchList.map(branch => ({ env, branch })));
|
||||
wsBranches = io(WS_BRANCHES);
|
||||
wsEnv = io(WS_ENVIRONMENT);
|
||||
wsErr = io(WS_ERRORS);
|
||||
|
||||
wsBranches.on('branches', (data) => {
|
||||
branches = Object.entries(data).flatMap(([env, branchList]) => branchList.map(branch => ({ env, branch })));
|
||||
renderBranches();
|
||||
showStatus('Branches updated via Socket.IO.');
|
||||
});
|
||||
|
||||
socket.on('environments', (data) => {
|
||||
wsEnv.on('environments', (data) => {
|
||||
environment = data;
|
||||
renderBranches();
|
||||
showStatus('Environment updated via Socket.IO.');
|
||||
});
|
||||
|
||||
socket.on('error', (data) => {
|
||||
wsErr.on('error', (data) => {
|
||||
showStatus(data.message || 'Unknown error', true);
|
||||
});
|
||||
}
|
||||
|
||||
function sendEnvironmentUpdate(envUpdate) {
|
||||
socket.emit('update', envUpdate);
|
||||
wsEnv.emit('update', envUpdate);
|
||||
}
|
||||
|
||||
function updateEnvironment() {
|
||||
|
||||
Reference in New Issue
Block a user