diff --git a/AI.instructions.md b/AI.instructions.md index 591e045..8aa7da4 100644 --- a/AI.instructions.md +++ b/AI.instructions.md @@ -1,3 +1,11 @@ +docker run -it --rm \ +-v /media/DATA/src/brencher:/br \ +-v /var/run/docker.sock:/var/run/docker.sock \ +registry.rudolf.keenetic.link/brencher:auto-b62e6 \ +/bin/bash + + + # Startup config .env - Git repository url - Authentication username diff --git a/README.instruction.md b/README.instruction.md index cb858c4..6cf4e32 100644 --- a/README.instruction.md +++ b/README.instruction.md @@ -2,6 +2,7 @@ Use language python. + Locally run in venv create with uv tool. Use venv with uv for Python environment management and dependency installation. Keep requirements.txt updated. diff --git a/application.conf b/application.conf deleted file mode 100644 index 4ba9b37..0000000 --- a/application.conf +++ /dev/null @@ -1,13 +0,0 @@ -ktor { - deployment { - port = 8080 - } - application { - modules = [ org.rudolf.ApplicationKt.module ] - } - include required(classpath("local.conf")) - git { - repositoryUrl = "https://github.com/rudolf1/brencher.git" - branchRefreshIntervalMinutes = 5 - } -} diff --git a/backend/app.py b/backend/app.py index da6f83d..36ed58d 100644 --- a/backend/app.py +++ b/backend/app.py @@ -21,6 +21,8 @@ logger = logging.getLogger(__name__) # Load environment variables from .env file load_dotenv("local.env") +load_dotenv('/run/secrets/brencher-secrets') + app = Flask(__name__, static_folder='frontend') socketio = SocketIO(app, cors_allowed_origins="*") diff --git a/backend/configs/brencher.py b/backend/configs/brencher.py index c869a46..f3460f4 100644 --- a/backend/configs/brencher.py +++ b/backend/configs/brencher.py @@ -8,7 +8,7 @@ from steps.step import AbstractStep env = Environment( id="brencher", - branches=["main"], + branches=["main", "small_fix"], state="Active", repo="https://github.com/rudolf1/brencher.git", ) @@ -16,19 +16,29 @@ env = Environment( def create_pipeline(env: Environment) -> List[AbstractStep]: clone = GitClone(env) - checkoutMerged = CheckoutMerged(clone, env.branches, env=env) + checkoutMerged = CheckoutMerged(clone, env.branches, env=env, + push = False, + git_user_email="rudolfss13@gmail.com", + git_user_name="brencher_bot" + ) + +# git config --global user.email "rudolfss13@gmail.com" +# git config --global user.name "brencher_bot" + + buildDocker = DockerComposeBuild(clone, docker_repo_username = "", docker_repo_password = "", docker_compose_path = "docker-compose.yml", docker_repo_url="https://registry.rudolf.keenetic.link", publish=False, - envs = lambda: { "version": "auto-" + checkoutMerged.result[1][0:5] }, + envs = lambda: { "version": "auto-" + checkoutMerged.result.version }, env=env ) deployDocker = DockerSwarmDeploy( wd=clone, - envs = lambda: { "version": "auto-" + checkoutMerged.result[1][0:5] }, + buildDocker=buildDocker, + envs = lambda: { "version": "auto-" + checkoutMerged.result.version }, stack_name = "brencher", docker_compose_path = "docker-compose.yml", env=env, @@ -37,7 +47,12 @@ def create_pipeline(env: Environment) -> List[AbstractStep]: clone, checkoutMerged, buildDocker, - deployDocker + deployDocker, + # TODO add health check. + # TODO add reverse flow. We need to understand what is deployed + # TODO add teamcity support + # TODO add udploy support + ] brencher: Tuple[Environment, List[AbstractStep]] = (env, create_pipeline(env)) diff --git a/backend/steps/docker.py b/backend/steps/docker.py index f1df21d..8201b52 100644 --- a/backend/steps/docker.py +++ b/backend/steps/docker.py @@ -37,19 +37,20 @@ class DockerComposeBuild(AbstractStep[List[str]]): images_built = [] try: # Authenticate to docker repo - client = docker.from_env() + client = docker.DockerClient(base_url='unix://var/run/docker.sock') if self.publish: client.login(username=self.docker_repo_username, password=self.docker_repo_password, registry=self.docker_repo_url) env = self.envs() # Parse docker-compose file - with open(self.docker_compose_path, 'r') as f: + docker_compose_absolute_path = os.path.join(self.wd.result, self.docker_compose_path) + with open(docker_compose_absolute_path, 'r') as f: content = f.read() content = re.sub(r'\$\{([^}]+)\}', lambda m: env[m.group(1)], content) compose = yaml.safe_load(content) services = compose.get('services', {}) for name, svc in services.items(): - build_ctx = svc.get('build') + build_ctx = os.path.join(self.wd.result, svc.get('build')) image = svc.get('image') if not build_ctx or not image: continue @@ -82,54 +83,99 @@ class DockerComposeBuild(AbstractStep[List[str]]): except Exception as e: raise e -class DockerSwarmDeploy(AbstractStep): +class DockerSwarmDeploy(AbstractStep[str]): def __init__(self, wd: GitClone, + buildDocker: DockerComposeBuild, docker_compose_path: str, envs: Callable[[], Dict[str, str]], stack_name: str, **kwargs): super().__init__(**kwargs) self.wd = wd + self.buildDocker = buildDocker self.envs = envs self.docker_compose_path = docker_compose_path self.stack_name = stack_name - def progress(self) -> None: + def _get_stack_state(self, stack_name): + current_services = {} + cmd_ps = [ + "docker", "service", "ls", "--format", "{{.Name}}" + ] + ps_result = subprocess.run(cmd_ps, capture_output=True, text=True, check=True) + stack_svc_names = ps_result.stdout.strip().splitlines() + for svc_name in stack_svc_names: + if not svc_name.startswith(stack_name + "_"): + continue + cmd_inspect = [ + "docker", "service", "inspect", svc_name, "--format", "{{.Spec.TaskTemplate.ContainerSpec.Image}}" + ] + inspect_result = subprocess.run(cmd_inspect, capture_output=True, text=True, check=True) + image_tag = inspect_result.stdout.strip() + service_name = svc_name.removeprefix(self.stack_name + "_") + current_services[service_name] = image_tag + logger.info(f"Current services {current_services}") + return current_services + + def progress(self) -> str: """ Deploys to Docker Swarm using the specified docker-compose.yaml. """ if isinstance(self.wd.result, BaseException): raise self.wd.result - try: - env = self.envs() - # Prepare docker-compose file with env substitution - docker_compose_absolute_path = os.path.join(self.wd.result, self.docker_compose_path) - with open(docker_compose_absolute_path, 'r') as f: - content = f.read() - content = re.sub(r'\$\{([^}]+)\}', lambda m: env.get(m.group(1), ""), content) - compose = yaml.safe_load(content) - if "services" in compose: - for svc in compose["services"].values(): - if "build" in svc: - del svc["build"] - content = yaml.safe_dump(compose) - tmp_compose_path = docker_compose_absolute_path + ".tmp" - with open(tmp_compose_path, 'w') as f: - f.write(content) + if isinstance(self.buildDocker.result, BaseException): + raise self.buildDocker.result - # Deploy stack to Docker Swarm - cmd = [ - "docker", "stack", "deploy", - "-c", tmp_compose_path, - self.stack_name - ] - logger.info(f"Deploying stack '{self.stack_name}' using {tmp_compose_path}") - result = subprocess.run(cmd, capture_output=True, text=True) - if result.returncode != 0: - logger.error(f"Stack deploy failed: {result.stderr}") - raise RuntimeError(f"Stack deploy failed: {result.stderr}") - logger.info(f"Stack deployed successfully: {result.stdout}") - except Exception as e: - logger.error(f"DockerSwarmDeploy failed: {e}") - raise + # TODO skip if correct stack already deployed + # Only image tags compared. Need more. + # TODO I guess label for container should be added. + # TODO Or find keys for docker swarm to skip deploy if no changes. + + env = self.envs() + # Prepare docker-compose file with env substitution + docker_compose_absolute_path = os.path.join(self.wd.result, self.docker_compose_path) + with open(docker_compose_absolute_path, 'r') as f: + content = f.read() + content = re.sub(r'\$\{([^}]+)\}', lambda m: env.get(m.group(1), ""), content) + compose = yaml.safe_load(content) + if "services" in compose: + for svc in compose["services"].values(): + if "build" in svc: + del svc["build"] + content = yaml.safe_dump(compose) + + current_services = self._get_stack_state(self.stack_name) + expected_services = compose.get("services", {}) + diffs = [] + ok = [] + for svc_name, svc in expected_services.items(): + expected_image = svc.get("image") + running_image = current_services.get(svc_name) + l = f"Service '{svc_name}' (expected image: '{expected_image}', actual image: '{running_image}')" + logger.info(l) + if running_image and expected_image and running_image == expected_image: + ok.append(l) + else: + diffs.append(l) + + if len(diffs) == 0: + return '\\n'.join(ok) + logger.info(f"Diff {diffs}") + tmp_compose_path = docker_compose_absolute_path + ".tmp" + with open(tmp_compose_path, 'w') as f: + f.write(content) + + # Deploy stack to Docker Swarm + cmd = [ + "docker", "stack", "deploy", + "-c", tmp_compose_path, + self.stack_name + ] + logger.info(f"Deploying stack '{self.stack_name}' using {tmp_compose_path}") + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + logger.error(f"Stack deploy failed: {result.stderr}") + raise RuntimeError(f"Stack deploy failed: {result.stderr}") + logger.info(f"Stack deployed successfully: {result.stdout}") + return f"Stack deployed successfully: {result.stdout}" diff --git a/backend/steps/git.py b/backend/steps/git.py index efe5df2..4c78c9f 100644 --- a/backend/steps/git.py +++ b/backend/steps/git.py @@ -35,14 +35,27 @@ class GitClone(AbstractStep[str]): import hashlib -class CheckoutMerged(AbstractStep[Tuple[str, str]]): +@dataclass +class CheckoutAndMergeResult: + branch_name: str + commit_hash: str + version: str + +class CheckoutMerged(AbstractStep[CheckoutAndMergeResult]): wd: GitClone branches: List[str] - def __init__(self, wd: GitClone, branches: List[str], **kwargs): + def __init__(self, wd: GitClone, branches: List[str], + git_user_email, + git_user_name, + push: bool = True, + **kwargs): super().__init__(**kwargs) self.wd = wd self.branches = branches + self.git_user_email = git_user_email + self.git_user_name = git_user_name + self.push = push def _commits_tree(self, repo: git.Repo) -> Dict[Commit, List[Commit]]: childs: Dict[Commit, List[Commit]] = {} @@ -66,7 +79,7 @@ class CheckoutMerged(AbstractStep[Tuple[str, str]]): return visited - def progress(self) -> Tuple[str, str]: + def progress(self) -> CheckoutAndMergeResult: if self.wd.result_obj is BaseException: raise self.wd.result_obj @@ -79,22 +92,28 @@ class CheckoutMerged(AbstractStep[Tuple[str, str]]): # Open the repository repo = git.Repo(repo_path) - + # Set user email and name for the repo + with repo.config_writer() as cw: + cw.set_value("user", "email", self.git_user_email) + cw.set_value("user", "name", self.git_user_name) logger.info(f"Selected branches: {self.branches}") # Extract commit ids for the selected branches commit_ids: Dict[Commit, str] = {} - for branch in self.branches: - commit = repo.commit(f'origin/{branch}') - commit_ids[commit] = branch + for br in self.branches: + commit = repo.commit(f'origin/{br}') + commit_ids[commit] = br + + logger.info(f"Commit ids for branches: {commit_ids}") tree = self._commits_tree(repo) merge_commit = [(c, self._find_merge_childs(tree, c)) for c in commit_ids.keys()] - result = merge_commit[0] + result = set(merge_commit[0][1]) for c, l in merge_commit[1:]: - inters = set(result).intersection(set(l)) - if len(inters) == 0: + result = set(result).intersection(set(l)) + if len(result) == 0: break + if len(result) > 0: merge_commit = [c for c in merge_commit[0][1] if c in result][0] logger.info(f"Common commit found {merge_commit}") @@ -102,24 +121,30 @@ class CheckoutMerged(AbstractStep[Tuple[str, str]]): merge_commit = None logger.info(f"Common commit not found") - if merge_commit is not None: - for head in repo.branches: - if head.commit.hexsha == merge_commit.hexsha: - logger.info(f"Merge commit {merge_commit} corresponds to branch {head}") - return (head.name, merge_commit.hexsha) sorted_commits = sorted(commit_ids.keys(), key=lambda x: x.hexsha) auto_branch_hash = hashlib.sha1(''.join([x.hexsha for x in sorted_commits]).encode()).hexdigest() - logger.info(f"Commit ids for branches: {commit_ids}") - auto_branch_name = f"auto/{auto_branch_hash}" + version = '-'.join([x.hexsha[0:5] for x in sorted_commits]) + if merge_commit is not None: + for head in repo.branches: + if head.is_remote and head.commit.hexsha == merge_commit.hexsha: + logger.info(f"Merge commit {merge_commit} corresponds to branch {head}") + return CheckoutAndMergeResult(head.name, merge_commit.hexsha, version) + # TODO No sense to generate branch name from hash. Just use commit it after merge. + # TODO Not sure. Think about it. + # TODO Write test to cover usecases. + # TODO Blocker!!! Not able to push in docker. + # TODO Blocker!!! Do not lookup local branches. + if merge_commit is not None: repo.git.branch('-f', auto_branch_name, merge_commit.hexsha) repo.git.checkout(auto_branch_name) - repo.git.push('-f', 'origin', auto_branch_name) logger.info(f"Head is {repo.head.commit}") - logger.info(f"Pushed {merge_commit} to {auto_branch_name}") - return (auto_branch_name, merge_commit.hexsha) + if self.push: + repo.git.push('-f', 'origin', auto_branch_name) + logger.info(f"Pushed {merge_commit} to {auto_branch_name}") + return CheckoutAndMergeResult(auto_branch_name, merge_commit.hexsha, version) logger.info(f"Looking for existing auto branch: {auto_branch_name}") @@ -131,40 +156,35 @@ class CheckoutMerged(AbstractStep[Tuple[str, str]]): repo.git.checkout(auto_branch_name) auto_branch_hash = repo.head.commit.hexsha - return (auto_branch_name, auto_branch_hash) + return CheckoutAndMergeResult(auto_branch_name, auto_branch_hash, version) + repo.git.checkout(repo.head.commit.hexsha) + if auto_branch_name in repo.branches: + repo.git.branch('-D', auto_branch_name) # Auto branch doesn't exist, create it logger.info(f"Creating new auto branch: {auto_branch_name}") - # Start with the first branch - base_branch = self.branches[0] - repo.git.checkout(base_branch) - - # Create temporary branch for merging - temp_branch = f"temp-merge-{auto_branch_hash}" - # Remove temp_branch if it exists - if temp_branch in repo.heads: - repo.git.branch('-D', temp_branch) - repo.git.checkout('-b', temp_branch) + commits = list(commit_ids.keys()) + repo.git.branch('-f', auto_branch_name, commits[0].hexsha) + repo.git.checkout(auto_branch_name) # Merge the rest of the branches - for branch in self.branches[1:]: + for commit in commits[1:]: try: - logger.info(f"Merging branch: {branch}") - repo.git.merge(branch, '--no-ff') + logger.info(f"Merging commit: {commit}") + repo.git.merge(commit, '--no-ff') except git.GitCommandError as e: # Handle merge conflicts according to predefined rules # For now, we'll abort the merge and report failure repo.git.merge('--abort') - error_message = f"Merge conflict when merging {branch}: {str(e)}" + error_message = f"Merge conflict when merging {commit}: {str(e)}" logger.error(error_message) raise BaseException(error_message) - # Create and push the auto branch - repo.git.branch('-f', auto_branch_name) - repo.git.checkout(auto_branch_name) - auto_branch_hash = repo.head.commit.hexsha - repo.git.push('-f', 'origin', auto_branch_name) - return (auto_branch_name,auto_branch_hash) + auto_branch_hash = repo.head.commit.hexsha + if self.push: + logger.info(f"Pushing {auto_branch_hash} to {auto_branch_name}") + repo.git.push('-f', 'origin', auto_branch_name) + return CheckoutAndMergeResult(auto_branch_name,auto_branch_hash, version) diff --git a/docker-compose.yml b/docker-compose.yml index ca89911..5f70327 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,10 +3,13 @@ services: brencher-backend: build: . image: registry.rudolf.keenetic.link/brencher:${version} + user: 1000:998 ports: - "5000:5000" + container_name: brencher-backend volumes: - /var/run/docker.sock:/var/run/docker.sock + - /etc/passwd:/etc/passwd environment: - PYTHONUNBUFFERED=1 deploy: @@ -15,6 +18,12 @@ services: condition: on-failure networks: - brencher-net + secrets: + - brencher-secrets + +secrets: + brencher-secrets: + external: true networks: brencher-net: