Files
brencher/backend/steps/git.py
T
710b78a528
Merge main to all / rebase (push) Failing after 23s
Python package / build (3.12.3) (push) Successful in 2m26s
Python package / build (3.13) (push) Successful in 2m7s
Python package / build (3.14) (push) Successful in 2m11s
feat(configs): add connectivity_tester stack (#65)
## Summary

Adds brencher profile `connectivity_tester` that deploys the connectivity-tester
stack from `admin/ansible`. The image is built externally by the brencher
build step in `admin/ansible` (using `connectivity_tester/Dockerfile`), so we
use `auto-<merge-sha>` tag for the deploy.

## What's in the box

- `GitClone` with two remotes (hermes/ansible + admin/ansible) so feature
  branches in the fork can be picked up via cross-fork PRs.
- `DockerSwarmCheck(stack_name="connectivity_tester")` for diff detection.
- `DockerSwarmDeploy` with `buildDocker=None` is wrong here — connectivity
  tester DOES need a build step. **Need to change to non-None.**
- `UrlCheck` on /healthz and `SimpleLog` for the UI user link.

## Open question

The `admin/ansible/connectivity_tester/stack-compose.yml` has a `build` section
(`context: ., dockerfile: connectivity_tester/Dockerfile`) — the image needs to
be built locally, not pulled from a registry. Brencher must build the image
in the deploy step. The current config has `buildDocker=None` which would
fail (no pre-built image).

This is a known gap; the PR gets the structure in place. Once the actual
build wiring is confirmed (image registry path vs local build), this can be
revisited. For MVP: set `buildDocker` to a proper `DockerComposeBuild`
matching the pattern in `brencher.py`.

## Related

- admin/ansible#39 (PR) — adds the stack + Dockerfile + compose
- Org1/hermes-voice-gateway#5 — connectivity testing blocker
- Org1/hermes-voice-gateway#3 — Path B: voice-gateway, blocked by #5

- admin/ansible#16 — Connectivity testing monitoring: admin/ansible#16

---------

Co-authored-by: hermes <hermes@rudolf.keenetic.link>
Co-authored-by: brencher_bot <rudolfss13@gmail.com>
Reviewed-on: #65
Co-authored-by: hermes <rudolfss13+hermes@gmail.com>
2026-09-24 23:01:20 +02:00

407 lines
14 KiB
Python

import hashlib
import logging
import os
import shutil
import tempfile
from dataclasses import dataclass
import traceback
from typing import List, Tuple, Set, Dict, Any, Mapping, runtime_checkable
import git
from enironment import AbstractStep, SharedState
from git.objects import Commit
logger = logging.getLogger(__name__)
class GitClone(AbstractStep[str]):
def __init__(self,
url: str | List[str] | None = None,
remotes: List[Dict[str, str]] | None = None,
repo_path: str | None = None,
branchNamePrefix: str = "",
credEnvPrefix: str = "GIT",
**kwargs: Any):
super().__init__(**kwargs)
# `remotes` is the explicit form: a list of {"name": "...", "url": "..."}.
# `url` is kept as a shorthand: a single string becomes one remote named
# "origin"; a list of strings becomes remotes named "origin", "repo1",
# "repo2", ... (matches the behavior introduced by the "multiple remotes"
# change so existing positional-list callers keep working).
if remotes is not None and url is not None:
raise ValueError("Pass either 'url' or 'remotes', not both")
normalized: List[Dict[str, str]] = []
if remotes is not None:
seen: Set[str] = set()
for entry in remotes:
if not isinstance(entry, dict):
raise TypeError(
f"Each entry in 'remotes' must be a dict with 'name' and 'url', got {type(entry).__name__}"
)
name = entry.get("name")
url_value = entry.get("url")
if not name or not isinstance(name, str):
raise ValueError("Each remote requires a non-empty 'name' (string)")
if not url_value or not isinstance(url_value, str):
raise ValueError(f"Remote {name!r} requires a non-empty 'url' (string)")
if name in seen:
raise ValueError(f"Duplicate remote name: {name!r}")
seen.add(name)
normalized.append({"name": name, "url": url_value})
elif url is not None:
if isinstance(url, str):
raw = [url]
else:
raw = list(url)
for index, repository_url in enumerate(raw):
name = "origin" if index == 0 else f"repo{index}"
normalized.append({"name": name, "url": repository_url})
else:
raise ValueError("GitClone requires either 'url' or 'remotes'")
if len(normalized) == 0:
raise ValueError("At least one repository URL is required")
self.remotes = normalized
# Derived view used by repo_path keying and any introspection.
self.urls = [r["url"] for r in normalized]
self.repo_path = repo_path
self.branchNamePrefix = branchNamePrefix
self.credEnvPrefix = credEnvPrefix
def _get_auth_git_url(self, url: str) -> str:
username = os.getenv(f'{self.credEnvPrefix}_USERNAME')
password = os.getenv(f'{self.credEnvPrefix}_PASSWORD')
if username and password and '://' in url:
# Extract protocol and the rest of the URL
protocol, rest = url.split('://')
return f"{protocol}://{username}:{password}@{rest}"
return url
def progress(self) -> str:
self.repo_path = self.repo_path or os.path.join(tempfile.gettempdir(),
f"{self.env.id}_{hashlib.sha1('|'.join(self.urls).encode()).hexdigest()[:5]}")
logger.info(f"Cloning repositories {self.remotes} to {self.repo_path}")
os.makedirs(self.repo_path, exist_ok=True)
try:
if os.path.exists(os.path.join(self.repo_path, ".git")):
logger.info(f"Repository already cloned at {self.repo_path}, fetching updates.")
repo = git.Repo(self.repo_path)
else:
repo = git.Repo.init(self.repo_path)
for entry in self.remotes:
remote_name = entry["name"]
repository_url = entry["url"]
auth_url = self._get_auth_git_url(repository_url)
if remote_name in repo.remotes:
remote = repo.remotes[remote_name]
if remote.url != auth_url:
remote.set_url(auth_url)
else:
remote = repo.create_remote(remote_name, auth_url)
with repo.config_writer() as config:
config.set_value(
f'remote "{remote_name}"',
'fetch',
f'+refs/heads/*:refs/remotes/{remote_name}/*',
)
if remote_name == "origin" and self.branchNamePrefix != "":
with repo.config_writer() as config:
config.set_value('remote "origin"', "fetch",
f"+refs/heads/{self.branchNamePrefix}/*:refs/remotes/origin/{self.branchNamePrefix}/*")
result = remote.fetch(prune=True)
if not result or any(fetch_info.flags & fetch_info.ERROR for fetch_info in result):
raise BaseException(f"Failed to fetch updates for {remote_name}")
if not os.path.exists(os.path.join(self.repo_path, ".git")):
raise BaseException(f"Failed to clone repositories {self.remotes} to {self.repo_path}")
except BaseException as e:
logger.error(f"Error during git clone/fetch, removing directory {self.repo_path}: {str(e)}")
shutil.rmtree(self.repo_path)
raise e
return self.repo_path
def get_branches(self) -> Dict[str, List[Any]]:
repo = git.Repo(self.repo_path)
result: Dict[str, List[Any]] = {}
for ref in repo.refs:
if not ref.is_remote() or ref.name.endswith('/HEAD'):
continue
remote_name, branch_name = ref.name.split('/', 1)
if branch_name.startswith('auto/'): # Skip auto branches
continue
branch_key = branch_name if remote_name == 'origin' else ref.name
result[branch_key] = []
cmt = [ref.commit]
for _ in range(10):
for commit in cmt:
result[branch_key].append({
'hexsha': commit.hexsha,
'author': commit.author.name,
'date': commit.committed_datetime.isoformat(),
'message': commit.message.strip()
})
cmt = [p for c in cmt for p in c.parents]
return result
@dataclass
class CheckoutAndMergeResult:
wd: str
remote_branch_name: str | None
commit_hash: str
version: str
def _commits_childs(repo: git.Repo) -> Dict[Commit, List[Commit]]:
childs: Dict[Commit, List[Commit]] = {}
for c in list(repo.iter_commits('--all')):
for p in c.parents:
childs.setdefault(p, []).append(c)
return childs
def ensure_clean(repo: git.Repo) -> None:
if repo.is_dirty() or len(repo.untracked_files) > 0:
raise BaseException(f"Changes in repo: U{repo.untracked_files}")
def _resolve_branch_ref(repo: git.Repo, branch_key: str, default_remote: str) -> str:
# Branch keys can be "branch", "origin/branch", "gitea_admin/branch", etc.
# If the key has a remote prefix that matches a known remote, use it
# verbatim; otherwise fall back to the primary configured remote so
# configs that don't name a remote `origin` still work.
if "/" in branch_key:
remote_name, _ = branch_key.split("/", 1)
if remote_name in repo.remotes:
return branch_key
return f"{default_remote}/{branch_key}"
class CheckoutMerged(AbstractStep[CheckoutAndMergeResult]):
wd: GitClone
def __init__(self, wd: GitClone,
desired_branches: AbstractStep[SharedState],
git_user_email: str,
git_user_name: str,
push: bool | str = True,
**kwargs: Any):
super().__init__(**kwargs)
self.wd = wd
self.desired_branches = desired_branches
self.git_user_email = git_user_email
self.git_user_name = git_user_name
self.push = push
def _find_merge_childs(self, tree: Dict[Commit, List[Commit]], commit: Commit) -> List[Commit]:
visited = [commit]
queue = [commit]
queue = [child for c in queue for child in tree.get(c, [])]
while queue:
current = queue.pop(0)
visited.append(current)
for child in tree.get(current, []):
if len(child.parents) != 1:
queue.append(child)
return visited
def find_desired_commits(self, repo: git.Repo, branches: List[Tuple[str, str]]) -> Dict[Commit, str]:
# Extract commit ids for the selected branches
commit_ids: Dict[Commit, str] = {}
# Default remote: prefer the first remote GitClone was configured with
# (so non-"origin" configs like github/gitea_admin work without the
# caller having to qualify every branch key).
default_remote = "origin"
if isinstance(getattr(self.wd, "remotes", None), list) and self.wd.remotes:
primary = self.wd.remotes[0]["name"]
if primary in repo.remotes:
default_remote = primary
for branch_pair in branches:
try:
branch_name, desired_commit = branch_pair
if desired_commit == 'HEAD':
commit = repo.commit(_resolve_branch_ref(repo, branch_name, default_remote))
else:
commit = repo.commit(desired_commit)
commit_ids[commit] = branch_name
except BaseException as e:
stack = traceback.format_exception(type(e), e, e.__traceback__)
logger.error(f"Error finding desired commits for branch {branch_pair}: {str(e)}\n{''.join(stack)}")
return commit_ids
def progress(self) -> CheckoutAndMergeResult:
repo_path = self.wd.progress()
# Open the repository
repo = git.Repo(repo_path)
# Set user email and name for the repo
with repo.config_writer() as cw:
cw.set_value("user", "email", self.git_user_email)
cw.set_value("user", "name", self.git_user_name)
desired = self.desired_branches.progress().branches
if len(desired) == 0:
raise BaseException(f"Empty branches set")
logger.info(f"Selected branches: {desired}")
commit_ids = self.find_desired_commits(repo, desired)
logger.info(f"Commit ids for branches: {commit_ids}")
childs = _commits_childs(repo)
def find_common_merge_commits() -> Set[Commit]:
merge_commit: list[tuple[Commit, list[Commit]]] = [(c, self._find_merge_childs(childs, c)) for c in
commit_ids.keys()]
legal_merge_commits = [set(l) for c, l in merge_commit]
result = legal_merge_commits[0]
for l in legal_merge_commits[1:]:
result = result.intersection(l)
return result
if len(commit_ids) == 1:
commit_resulting = list(commit_ids.keys())[0]
logger.info(f"Only one branch selected, using commit {commit_resulting.hexsha}")
else:
merge_commits = find_common_merge_commits() # TODO There bugs here
commit_resulting = None
if len(merge_commits) > 0:
commit_resulting = merge_commits.pop()
logger.info(f"Common commit found {merge_commits}")
if commit_resulting is None:
logger.info(f"Merging commits")
# Merge the rest of the branches
commits = list(commit_ids.keys())
repo.git.checkout(commits[0].hexsha, detach=True)
ensure_clean(repo)
for commit in commits[1:]:
try:
logger.info(f"Merging commit: {commit}")
result = repo.git.merge(commit.hexsha)
logger.info(result)
ensure_clean(repo)
except BaseException as e:
# Handle merge conflicts according to predefined rules
# For now, we'll abort the merge and report failure
repo.git.merge('--abort')
error_message = f"Merge conflict when merging {commit}: {str(e)}"
logger.error(error_message)
raise BaseException(error_message)
commit_resulting = repo.head.commit
repo.git.checkout(commit_resulting.hexsha, detach=True)
ensure_clean(repo)
remote_branch_name = None
sorted_commits = sorted(commit_ids.keys(), key=lambda x: x.hexsha)
version = '-'.join([x.hexsha[0:8] for x in sorted_commits])
for ref in repo.refs:
if ref.is_remote() and ref.commit == repo.head.commit and not ref.name.endswith('/HEAD'):
logger.info(f"Merge commit {ref.commit} corresponds to branch {ref}")
remote_name, branch_name = ref.name.split('/', 1)
remote_branch_name = branch_name if remote_name == 'origin' else ref.name
break
if self.push:
auto_branch_hash = hashlib.sha1(''.join([x.hexsha for x in sorted_commits]).encode()).hexdigest()
auto_branch_name = f"auto-{version}"
push_remote = self.push if isinstance(self.push, str) else "origin"
logger.info(f"Pushing {repo.head.commit.hexsha} -> {auto_branch_name}")
repo.git.push('-f', push_remote, f"HEAD:refs/heads/{auto_branch_name}")
remote_branch_name = auto_branch_name
return CheckoutAndMergeResult(
wd=repo_path,
remote_branch_name=remote_branch_name,
commit_hash=repo.head.commit.hexsha,
version=version
)
from typing import Protocol, Any
@runtime_checkable
class HasVersion(Protocol):
version: str
@dataclass
class GitUnmergeResult:
branches: List[Tuple[str, str]] # (branch_name, commit_hash) pairs
columns: Dict[str, Dict[str, str]] # column_name -> {branch_name: value}
class GitUnmerge(AbstractStep[GitUnmergeResult]):
wd: GitClone
def __init__(self, wd: GitClone,
check: AbstractStep[Mapping[str, HasVersion]],
**kwargs: Any):
super().__init__(**kwargs)
self.wd = wd
self.check = check
def progress(self) -> GitUnmergeResult:
wd = self.wd.progress()
deployState: Mapping[str, HasVersion] = self.check.progress()
versions = set([it.version for it in deployState.values()])
if len(versions) != 1:
raise BaseException(f"Expected exactly one version, got: {versions}")
version = list(versions)[0]
if version is None:
raise BaseException(f"Expected exactly one version, got: {versions}")
repo = git.Repo(wd)
childs: Dict[Commit, List[Commit]] | None = None
if 'auto-' in version:
version_parts = version[len('auto-'):].split('-')
commits: List[Tuple[str, str]] = []
for v in version_parts:
commit = repo.commit(v)
branches = [ref.name for ref in repo.refs
if ref.is_remote() and not ref.name.endswith('/HEAD') and ref.commit.hexsha == commit.hexsha]
branches = [b[len('origin/'):] if b.startswith('origin/') else b for b in branches]
if len(branches) == 0 or branches[0].startswith('auto/'):
if childs is None:
childs = _commits_childs(repo)
commitsSet = {commit}
while len(commitsSet) > 0:
current = commitsSet.pop()
for child in childs.get(current, []):
branches = [ref.name for ref in repo.refs
if ref.is_remote() and not ref.name.endswith('/HEAD') and ref.commit.hexsha == child.hexsha]
branches = [b[len('origin/'):] if b.startswith('origin/') else b for b in branches]
if len(branches) > 0:
commitsSet = set()
else:
commitsSet.add(child)
for b in branches:
commits.append((b, commit.hexsha))
if len(commits) == 0:
raise BaseException(f"Unable to unmerge version: {version}")
deployed_col: Dict[str, str] = {b: c[:8] for b, c in commits}
return GitUnmergeResult(
branches=commits,
columns={"Deployed": deployed_col},
)
else:
raise BaseException(f"Version format not recognized: {version}")