before deploy to box

This commit is contained in:
2025-08-12 18:44:52 +02:00
parent 54941af8c9
commit 36347cf8b4
8 changed files with 183 additions and 95 deletions
+8
View File
@@ -1,3 +1,11 @@
docker run -it --rm \
-v /media/DATA/src/brencher:/br \
-v /var/run/docker.sock:/var/run/docker.sock \
registry.rudolf.keenetic.link/brencher:auto-b62e6 \
/bin/bash
# Startup config .env
- Git repository url
- Authentication username
+1
View File
@@ -2,6 +2,7 @@
Use language python.
Locally run in venv create with uv tool.
Use venv with uv for Python environment management and dependency installation.
Keep requirements.txt updated.
-13
View File
@@ -1,13 +0,0 @@
ktor {
deployment {
port = 8080
}
application {
modules = [ org.rudolf.ApplicationKt.module ]
}
include required(classpath("local.conf"))
git {
repositoryUrl = "https://github.com/rudolf1/brencher.git"
branchRefreshIntervalMinutes = 5
}
}
+2
View File
@@ -21,6 +21,8 @@ logger = logging.getLogger(__name__)
# Load environment variables from .env file
load_dotenv("local.env")
load_dotenv('/run/secrets/brencher-secrets')
app = Flask(__name__, static_folder='frontend')
socketio = SocketIO(app, cors_allowed_origins="*")
+20 -5
View File
@@ -8,7 +8,7 @@ from steps.step import AbstractStep
env = Environment(
id="brencher",
branches=["main"],
branches=["main", "small_fix"],
state="Active",
repo="https://github.com/rudolf1/brencher.git",
)
@@ -16,19 +16,29 @@ env = Environment(
def create_pipeline(env: Environment) -> List[AbstractStep]:
clone = GitClone(env)
checkoutMerged = CheckoutMerged(clone, env.branches, env=env)
checkoutMerged = CheckoutMerged(clone, env.branches, env=env,
push = False,
git_user_email="rudolfss13@gmail.com",
git_user_name="brencher_bot"
)
# git config --global user.email "rudolfss13@gmail.com"
# git config --global user.name "brencher_bot"
buildDocker = DockerComposeBuild(clone,
docker_repo_username = "",
docker_repo_password = "",
docker_compose_path = "docker-compose.yml",
docker_repo_url="https://registry.rudolf.keenetic.link",
publish=False,
envs = lambda: { "version": "auto-" + checkoutMerged.result[1][0:5] },
envs = lambda: { "version": "auto-" + checkoutMerged.result.version },
env=env
)
deployDocker = DockerSwarmDeploy(
wd=clone,
envs = lambda: { "version": "auto-" + checkoutMerged.result[1][0:5] },
buildDocker=buildDocker,
envs = lambda: { "version": "auto-" + checkoutMerged.result.version },
stack_name = "brencher",
docker_compose_path = "docker-compose.yml",
env=env,
@@ -37,7 +47,12 @@ def create_pipeline(env: Environment) -> List[AbstractStep]:
clone,
checkoutMerged,
buildDocker,
deployDocker
deployDocker,
# TODO add health check.
# TODO add reverse flow. We need to understand what is deployed
# TODO add teamcity support
# TODO add udploy support
]
brencher: Tuple[Environment, List[AbstractStep]] = (env, create_pipeline(env))
+82 -36
View File
@@ -37,19 +37,20 @@ class DockerComposeBuild(AbstractStep[List[str]]):
images_built = []
try:
# Authenticate to docker repo
client = docker.from_env()
client = docker.DockerClient(base_url='unix://var/run/docker.sock')
if self.publish:
client.login(username=self.docker_repo_username, password=self.docker_repo_password, registry=self.docker_repo_url)
env = self.envs()
# Parse docker-compose file
with open(self.docker_compose_path, 'r') as f:
docker_compose_absolute_path = os.path.join(self.wd.result, self.docker_compose_path)
with open(docker_compose_absolute_path, 'r') as f:
content = f.read()
content = re.sub(r'\$\{([^}]+)\}', lambda m: env[m.group(1)], content)
compose = yaml.safe_load(content)
services = compose.get('services', {})
for name, svc in services.items():
build_ctx = svc.get('build')
build_ctx = os.path.join(self.wd.result, svc.get('build'))
image = svc.get('image')
if not build_ctx or not image:
continue
@@ -82,54 +83,99 @@ class DockerComposeBuild(AbstractStep[List[str]]):
except Exception as e:
raise e
class DockerSwarmDeploy(AbstractStep):
class DockerSwarmDeploy(AbstractStep[str]):
def __init__(self,
wd: GitClone,
buildDocker: DockerComposeBuild,
docker_compose_path: str,
envs: Callable[[], Dict[str, str]],
stack_name: str,
**kwargs):
super().__init__(**kwargs)
self.wd = wd
self.buildDocker = buildDocker
self.envs = envs
self.docker_compose_path = docker_compose_path
self.stack_name = stack_name
def progress(self) -> None:
def _get_stack_state(self, stack_name):
current_services = {}
cmd_ps = [
"docker", "service", "ls", "--format", "{{.Name}}"
]
ps_result = subprocess.run(cmd_ps, capture_output=True, text=True, check=True)
stack_svc_names = ps_result.stdout.strip().splitlines()
for svc_name in stack_svc_names:
if not svc_name.startswith(stack_name + "_"):
continue
cmd_inspect = [
"docker", "service", "inspect", svc_name, "--format", "{{.Spec.TaskTemplate.ContainerSpec.Image}}"
]
inspect_result = subprocess.run(cmd_inspect, capture_output=True, text=True, check=True)
image_tag = inspect_result.stdout.strip()
service_name = svc_name.removeprefix(self.stack_name + "_")
current_services[service_name] = image_tag
logger.info(f"Current services {current_services}")
return current_services
def progress(self) -> str:
"""
Deploys to Docker Swarm using the specified docker-compose.yaml.
"""
if isinstance(self.wd.result, BaseException):
raise self.wd.result
try:
env = self.envs()
# Prepare docker-compose file with env substitution
docker_compose_absolute_path = os.path.join(self.wd.result, self.docker_compose_path)
with open(docker_compose_absolute_path, 'r') as f:
content = f.read()
content = re.sub(r'\$\{([^}]+)\}', lambda m: env.get(m.group(1), ""), content)
compose = yaml.safe_load(content)
if "services" in compose:
for svc in compose["services"].values():
if "build" in svc:
del svc["build"]
content = yaml.safe_dump(compose)
tmp_compose_path = docker_compose_absolute_path + ".tmp"
with open(tmp_compose_path, 'w') as f:
f.write(content)
if isinstance(self.buildDocker.result, BaseException):
raise self.buildDocker.result
# Deploy stack to Docker Swarm
cmd = [
"docker", "stack", "deploy",
"-c", tmp_compose_path,
self.stack_name
]
logger.info(f"Deploying stack '{self.stack_name}' using {tmp_compose_path}")
result = subprocess.run(cmd, capture_output=True, text=True)
if result.returncode != 0:
logger.error(f"Stack deploy failed: {result.stderr}")
raise RuntimeError(f"Stack deploy failed: {result.stderr}")
logger.info(f"Stack deployed successfully: {result.stdout}")
except Exception as e:
logger.error(f"DockerSwarmDeploy failed: {e}")
raise
# TODO skip if correct stack already deployed
# Only image tags compared. Need more.
# TODO I guess label for container should be added.
# TODO Or find keys for docker swarm to skip deploy if no changes.
env = self.envs()
# Prepare docker-compose file with env substitution
docker_compose_absolute_path = os.path.join(self.wd.result, self.docker_compose_path)
with open(docker_compose_absolute_path, 'r') as f:
content = f.read()
content = re.sub(r'\$\{([^}]+)\}', lambda m: env.get(m.group(1), ""), content)
compose = yaml.safe_load(content)
if "services" in compose:
for svc in compose["services"].values():
if "build" in svc:
del svc["build"]
content = yaml.safe_dump(compose)
current_services = self._get_stack_state(self.stack_name)
expected_services = compose.get("services", {})
diffs = []
ok = []
for svc_name, svc in expected_services.items():
expected_image = svc.get("image")
running_image = current_services.get(svc_name)
l = f"Service '{svc_name}' (expected image: '{expected_image}', actual image: '{running_image}')"
logger.info(l)
if running_image and expected_image and running_image == expected_image:
ok.append(l)
else:
diffs.append(l)
if len(diffs) == 0:
return '\\n'.join(ok)
logger.info(f"Diff {diffs}")
tmp_compose_path = docker_compose_absolute_path + ".tmp"
with open(tmp_compose_path, 'w') as f:
f.write(content)
# Deploy stack to Docker Swarm
cmd = [
"docker", "stack", "deploy",
"-c", tmp_compose_path,
self.stack_name
]
logger.info(f"Deploying stack '{self.stack_name}' using {tmp_compose_path}")
result = subprocess.run(cmd, capture_output=True, text=True)
if result.returncode != 0:
logger.error(f"Stack deploy failed: {result.stderr}")
raise RuntimeError(f"Stack deploy failed: {result.stderr}")
logger.info(f"Stack deployed successfully: {result.stdout}")
return f"Stack deployed successfully: {result.stdout}"
+61 -41
View File
@@ -35,14 +35,27 @@ class GitClone(AbstractStep[str]):
import hashlib
class CheckoutMerged(AbstractStep[Tuple[str, str]]):
@dataclass
class CheckoutAndMergeResult:
branch_name: str
commit_hash: str
version: str
class CheckoutMerged(AbstractStep[CheckoutAndMergeResult]):
wd: GitClone
branches: List[str]
def __init__(self, wd: GitClone, branches: List[str], **kwargs):
def __init__(self, wd: GitClone, branches: List[str],
git_user_email,
git_user_name,
push: bool = True,
**kwargs):
super().__init__(**kwargs)
self.wd = wd
self.branches = branches
self.git_user_email = git_user_email
self.git_user_name = git_user_name
self.push = push
def _commits_tree(self, repo: git.Repo) -> Dict[Commit, List[Commit]]:
childs: Dict[Commit, List[Commit]] = {}
@@ -66,7 +79,7 @@ class CheckoutMerged(AbstractStep[Tuple[str, str]]):
return visited
def progress(self) -> Tuple[str, str]:
def progress(self) -> CheckoutAndMergeResult:
if self.wd.result_obj is BaseException:
raise self.wd.result_obj
@@ -79,22 +92,28 @@ class CheckoutMerged(AbstractStep[Tuple[str, str]]):
# Open the repository
repo = git.Repo(repo_path)
# Set user email and name for the repo
with repo.config_writer() as cw:
cw.set_value("user", "email", self.git_user_email)
cw.set_value("user", "name", self.git_user_name)
logger.info(f"Selected branches: {self.branches}")
# Extract commit ids for the selected branches
commit_ids: Dict[Commit, str] = {}
for branch in self.branches:
commit = repo.commit(f'origin/{branch}')
commit_ids[commit] = branch
for br in self.branches:
commit = repo.commit(f'origin/{br}')
commit_ids[commit] = br
logger.info(f"Commit ids for branches: {commit_ids}")
tree = self._commits_tree(repo)
merge_commit = [(c, self._find_merge_childs(tree, c)) for c in commit_ids.keys()]
result = merge_commit[0]
result = set(merge_commit[0][1])
for c, l in merge_commit[1:]:
inters = set(result).intersection(set(l))
if len(inters) == 0:
result = set(result).intersection(set(l))
if len(result) == 0:
break
if len(result) > 0:
merge_commit = [c for c in merge_commit[0][1] if c in result][0]
logger.info(f"Common commit found {merge_commit}")
@@ -102,24 +121,30 @@ class CheckoutMerged(AbstractStep[Tuple[str, str]]):
merge_commit = None
logger.info(f"Common commit not found")
if merge_commit is not None:
for head in repo.branches:
if head.commit.hexsha == merge_commit.hexsha:
logger.info(f"Merge commit {merge_commit} corresponds to branch {head}")
return (head.name, merge_commit.hexsha)
sorted_commits = sorted(commit_ids.keys(), key=lambda x: x.hexsha)
auto_branch_hash = hashlib.sha1(''.join([x.hexsha for x in sorted_commits]).encode()).hexdigest()
logger.info(f"Commit ids for branches: {commit_ids}")
auto_branch_name = f"auto/{auto_branch_hash}"
version = '-'.join([x.hexsha[0:5] for x in sorted_commits])
if merge_commit is not None:
for head in repo.branches:
if head.is_remote and head.commit.hexsha == merge_commit.hexsha:
logger.info(f"Merge commit {merge_commit} corresponds to branch {head}")
return CheckoutAndMergeResult(head.name, merge_commit.hexsha, version)
# TODO No sense to generate branch name from hash. Just use commit it after merge.
# TODO Not sure. Think about it.
# TODO Write test to cover usecases.
# TODO Blocker!!! Not able to push in docker.
# TODO Blocker!!! Do not lookup local branches.
if merge_commit is not None:
repo.git.branch('-f', auto_branch_name, merge_commit.hexsha)
repo.git.checkout(auto_branch_name)
repo.git.push('-f', 'origin', auto_branch_name)
logger.info(f"Head is {repo.head.commit}")
logger.info(f"Pushed {merge_commit} to {auto_branch_name}")
return (auto_branch_name, merge_commit.hexsha)
if self.push:
repo.git.push('-f', 'origin', auto_branch_name)
logger.info(f"Pushed {merge_commit} to {auto_branch_name}")
return CheckoutAndMergeResult(auto_branch_name, merge_commit.hexsha, version)
logger.info(f"Looking for existing auto branch: {auto_branch_name}")
@@ -131,40 +156,35 @@ class CheckoutMerged(AbstractStep[Tuple[str, str]]):
repo.git.checkout(auto_branch_name)
auto_branch_hash = repo.head.commit.hexsha
return (auto_branch_name, auto_branch_hash)
return CheckoutAndMergeResult(auto_branch_name, auto_branch_hash, version)
repo.git.checkout(repo.head.commit.hexsha)
if auto_branch_name in repo.branches:
repo.git.branch('-D', auto_branch_name)
# Auto branch doesn't exist, create it
logger.info(f"Creating new auto branch: {auto_branch_name}")
# Start with the first branch
base_branch = self.branches[0]
repo.git.checkout(base_branch)
# Create temporary branch for merging
temp_branch = f"temp-merge-{auto_branch_hash}"
# Remove temp_branch if it exists
if temp_branch in repo.heads:
repo.git.branch('-D', temp_branch)
repo.git.checkout('-b', temp_branch)
commits = list(commit_ids.keys())
repo.git.branch('-f', auto_branch_name, commits[0].hexsha)
repo.git.checkout(auto_branch_name)
# Merge the rest of the branches
for branch in self.branches[1:]:
for commit in commits[1:]:
try:
logger.info(f"Merging branch: {branch}")
repo.git.merge(branch, '--no-ff')
logger.info(f"Merging commit: {commit}")
repo.git.merge(commit, '--no-ff')
except git.GitCommandError as e:
# Handle merge conflicts according to predefined rules
# For now, we'll abort the merge and report failure
repo.git.merge('--abort')
error_message = f"Merge conflict when merging {branch}: {str(e)}"
error_message = f"Merge conflict when merging {commit}: {str(e)}"
logger.error(error_message)
raise BaseException(error_message)
# Create and push the auto branch
repo.git.branch('-f', auto_branch_name)
repo.git.checkout(auto_branch_name)
auto_branch_hash = repo.head.commit.hexsha
repo.git.push('-f', 'origin', auto_branch_name)
return (auto_branch_name,auto_branch_hash)
auto_branch_hash = repo.head.commit.hexsha
if self.push:
logger.info(f"Pushing {auto_branch_hash} to {auto_branch_name}")
repo.git.push('-f', 'origin', auto_branch_name)
return CheckoutAndMergeResult(auto_branch_name,auto_branch_hash, version)
+9
View File
@@ -3,10 +3,13 @@ services:
brencher-backend:
build: .
image: registry.rudolf.keenetic.link/brencher:${version}
user: 1000:998
ports:
- "5000:5000"
container_name: brencher-backend
volumes:
- /var/run/docker.sock:/var/run/docker.sock
- /etc/passwd:/etc/passwd
environment:
- PYTHONUNBUFFERED=1
deploy:
@@ -15,6 +18,12 @@ services:
condition: on-failure
networks:
- brencher-net
secrets:
- brencher-secrets
secrets:
brencher-secrets:
external: true
networks:
brencher-net: