Files
brencher/backend/steps/git.py
T
hermesandadmin 83b50af9b3
Merge main to all / rebase (push) Failing after 19s
Python package / build (3.12.3) (push) Successful in 2m5s
Python package / build (3.13) (push) Successful in 2m4s
Python package / build (3.14) (push) Successful in 2m9s
feat(ui): mark unhealthy environment toggles (#82)
## Use case

Environment toggle buttons get a red border when the environment pipeline contains a failed or unfinished step. The state updates with WebSocket environment updates and does not disable the toggle.

## Verification

- `node --check frontend/app.js`
- `git diff --check`
- `27 passed, 1 skipped`; 4 Playwright tests blocked by unavailable `/var/run/docker.sock`

---------

Co-authored-by: rudolf <rudolfss13@gmail.com>
Reviewed-on: #82
Co-authored-by: hermes <rudolfss13+hermes@gmail.com>
2026-10-07 23:03:16 +02:00

424 lines
15 KiB
Python

import hashlib
import logging
import os
import shutil
import tempfile
from dataclasses import dataclass
import traceback
from typing import List, Tuple, Set, Dict, Any, Mapping, runtime_checkable
import git
from enironment import AbstractStep, SharedState
from git.objects import Commit
logger = logging.getLogger(__name__)
AUTO_BRANCH_PREFIX = "auto-"
class GitClone(AbstractStep[str]):
def __init__(self,
url: str | List[str] | None = None,
remotes: List[Dict[str, str]] | None = None,
repo_path: str | None = None,
branchNamePrefix: str = "",
credEnvPrefix: str = "GIT",
**kwargs: Any):
super().__init__(**kwargs)
# `remotes` is the explicit form: a list of {"name": "...", "url": "..."}.
# `url` is kept as a shorthand: a single string becomes one remote named
# "origin"; a list of strings becomes remotes named "origin", "repo1",
# "repo2", ... (matches the behavior introduced by the "multiple remotes"
# change so existing positional-list callers keep working).
if remotes is not None and url is not None:
raise ValueError("Pass either 'url' or 'remotes', not both")
normalized: List[Dict[str, str]] = []
if remotes is not None:
seen: Set[str] = set()
for entry in remotes:
if not isinstance(entry, dict):
raise TypeError(
f"Each entry in 'remotes' must be a dict with 'name' and 'url', got {type(entry).__name__}"
)
name = entry.get("name")
url_value = entry.get("url")
if not name or not isinstance(name, str):
raise ValueError("Each remote requires a non-empty 'name' (string)")
if not url_value or not isinstance(url_value, str):
raise ValueError(f"Remote {name!r} requires a non-empty 'url' (string)")
if name in seen:
raise ValueError(f"Duplicate remote name: {name!r}")
seen.add(name)
normalized.append({"name": name, "url": url_value})
elif url is not None:
if isinstance(url, str):
raw = [url]
else:
raw = list(url)
for index, repository_url in enumerate(raw):
name = "origin" if index == 0 else f"repo{index}"
normalized.append({"name": name, "url": repository_url})
else:
raise ValueError("GitClone requires either 'url' or 'remotes'")
if len(normalized) == 0:
raise ValueError("At least one repository URL is required")
self.remotes = normalized
# Derived view used by repo_path keying and any introspection.
self.urls = [r["url"] for r in normalized]
self.repo_path = repo_path
self.branchNamePrefix = branchNamePrefix
self.credEnvPrefix = credEnvPrefix
self.errors: List[str] = []
def _get_auth_git_url(self, url: str) -> str:
username = os.getenv(f'{self.credEnvPrefix}_USERNAME')
password = os.getenv(f'{self.credEnvPrefix}_PASSWORD')
if username and password and '://' in url:
# Extract protocol and the rest of the URL
protocol, rest = url.split('://')
return f"{protocol}://{username}:{password}@{rest}"
return url
def progress(self) -> str:
self.errors = []
self.repo_path = self.repo_path or os.path.join(tempfile.gettempdir(),
f"{self.env.id}_{hashlib.sha1('|'.join(self.urls).encode()).hexdigest()[:5]}")
logger.info(f"Cloning repositories {self.remotes} to {self.repo_path}")
os.makedirs(self.repo_path, exist_ok=True)
try:
if os.path.exists(os.path.join(self.repo_path, ".git")):
logger.info(f"Repository already cloned at {self.repo_path}, fetching updates.")
repo = git.Repo(self.repo_path)
else:
repo = git.Repo.init(self.repo_path)
fetched_remotes = 0
for entry in self.remotes:
remote_name = entry["name"]
repository_url = entry["url"]
try:
auth_url = self._get_auth_git_url(repository_url)
if remote_name in repo.remotes:
remote = repo.remotes[remote_name]
if remote.url != auth_url:
remote.set_url(auth_url)
else:
remote = repo.create_remote(remote_name, auth_url)
with repo.config_writer() as config:
config.set_value(
f'remote "{remote_name}"',
'fetch',
f'+refs/heads/*:refs/remotes/{remote_name}/*',
)
if remote_name == "origin" and self.branchNamePrefix != "":
with repo.config_writer() as config:
config.set_value('remote "origin"', "fetch",
f"+refs/heads/{self.branchNamePrefix}/*:refs/remotes/origin/{self.branchNamePrefix}/*")
result = remote.fetch(prune=True)
if not result or any(fetch_info.flags & fetch_info.ERROR for fetch_info in result):
raise BaseException(f"Failed to fetch updates for {remote_name}")
fetched_remotes += 1
except BaseException as remote_error:
if "File contains no section headers" in str(remote_error):
raise
self.errors.append(f"Unable to fetch remote {remote_name} ({repository_url}): {remote_error}")
logger.warning("Unable to fetch remote %s (%s): %s", remote_name, repository_url, remote_error)
if fetched_remotes == 0:
logger.warning("No configured git remotes are available; keeping the local repository state")
if not os.path.exists(os.path.join(self.repo_path, ".git")):
raise BaseException(f"Failed to clone repositories {self.remotes} to {self.repo_path}")
except BaseException as e:
logger.error(f"Error during git clone/fetch, removing directory {self.repo_path}: {str(e)}")
shutil.rmtree(self.repo_path)
raise e
return self.repo_path
def get_branches(self) -> Dict[str, List[Any]]:
repo = git.Repo(self.repo_path)
result: Dict[str, List[Any]] = {}
for ref in repo.refs:
if not ref.is_remote() or ref.name.endswith('/HEAD'):
continue
remote_name, branch_name = ref.name.split('/', 1)
if branch_name.startswith(AUTO_BRANCH_PREFIX) or branch_name.startswith('auto/'):
continue
branch_key = branch_name if remote_name == 'origin' else ref.name
result[branch_key] = []
cmt = [ref.commit]
for _ in range(10):
for commit in cmt:
result[branch_key].append({
'hexsha': commit.hexsha,
'author': commit.author.name,
'date': commit.committed_datetime.isoformat(),
'message': commit.message.strip()
})
cmt = [p for c in cmt for p in c.parents]
return result
@dataclass
class CheckoutAndMergeResult:
wd: str
remote_branch_name: str | None
commit_hash: str
version: str
def _commits_childs(repo: git.Repo) -> Dict[Commit, List[Commit]]:
childs: Dict[Commit, List[Commit]] = {}
for c in list(repo.iter_commits('--all')):
for p in c.parents:
childs.setdefault(p, []).append(c)
return childs
def ensure_clean(repo: git.Repo) -> None:
if repo.is_dirty() or len(repo.untracked_files) > 0:
raise BaseException(f"Changes in repo: U{repo.untracked_files}")
def _resolve_branch_ref(repo: git.Repo, branch_key: str, default_remote: str) -> str:
# Branch keys can be "branch", "origin/branch", "gitea_admin/branch", etc.
# If the key has a remote prefix that matches a known remote, use it
# verbatim; otherwise fall back to the primary configured remote so
# configs that don't name a remote `origin` still work.
if "/" in branch_key:
remote_name, _ = branch_key.split("/", 1)
if remote_name in repo.remotes:
return branch_key
return f"{default_remote}/{branch_key}"
class CheckoutMerged(AbstractStep[CheckoutAndMergeResult]):
wd: GitClone
def __init__(self, wd: GitClone,
desired_branches: AbstractStep[SharedState],
git_user_email: str,
git_user_name: str,
push: bool | str = True,
**kwargs: Any):
super().__init__(**kwargs)
self.wd = wd
self.desired_branches = desired_branches
self.git_user_email = git_user_email
self.git_user_name = git_user_name
self.push = push
def _find_merge_childs(self, tree: Dict[Commit, List[Commit]], commit: Commit) -> List[Commit]:
visited = [commit]
queue = [commit]
queue = [child for c in queue for child in tree.get(c, [])]
while queue:
current = queue.pop(0)
visited.append(current)
for child in tree.get(current, []):
if len(child.parents) != 1:
queue.append(child)
return visited
def find_desired_commits(self, repo: git.Repo, branches: List[Tuple[str, str]]) -> Dict[Commit, str]:
# Extract commit ids for the selected branches
commit_ids: Dict[Commit, str] = {}
# Default remote: prefer the first remote GitClone was configured with
# (so non-"origin" configs like github/gitea_admin work without the
# caller having to qualify every branch key).
default_remote = "origin"
if isinstance(getattr(self.wd, "remotes", None), list) and self.wd.remotes:
primary = self.wd.remotes[0]["name"]
if primary in repo.remotes:
default_remote = primary
for branch_pair in branches:
try:
branch_name, desired_commit = branch_pair
if desired_commit == 'HEAD':
commit = repo.commit(_resolve_branch_ref(repo, branch_name, default_remote))
else:
commit = repo.commit(desired_commit)
commit_ids[commit] = branch_name
except BaseException as e:
stack = traceback.format_exception(type(e), e, e.__traceback__)
logger.error(f"Error finding desired commits for branch {branch_pair}: {str(e)}\n{''.join(stack)}")
return commit_ids
def progress(self) -> CheckoutAndMergeResult:
repo_path = self.wd.progress()
# Open the repository
repo = git.Repo(repo_path)
# Set user email and name for the repo
with repo.config_writer() as cw:
cw.set_value("user", "email", self.git_user_email)
cw.set_value("user", "name", self.git_user_name)
desired = self.desired_branches.progress().branches
if len(desired) == 0:
raise BaseException(f"Empty branches set")
logger.info(f"Selected branches: {desired}")
commit_ids = self.find_desired_commits(repo, desired)
logger.info(f"Commit ids for branches: {commit_ids}")
childs = _commits_childs(repo)
def find_common_merge_commits() -> Set[Commit]:
merge_commit: list[tuple[Commit, list[Commit]]] = [(c, self._find_merge_childs(childs, c)) for c in
commit_ids.keys()]
legal_merge_commits = [set(l) for c, l in merge_commit]
result = legal_merge_commits[0]
for l in legal_merge_commits[1:]:
result = result.intersection(l)
return result
if len(commit_ids) == 1:
commit_resulting = list(commit_ids.keys())[0]
logger.info(f"Only one branch selected, using commit {commit_resulting.hexsha}")
else:
merge_commits = find_common_merge_commits() # TODO There bugs here
commit_resulting = None
if len(merge_commits) > 0:
commit_resulting = merge_commits.pop()
logger.info(f"Common commit found {merge_commits}")
if commit_resulting is None:
logger.info(f"Merging commits")
# Merge the rest of the branches
commits = list(commit_ids.keys())
repo.git.checkout(commits[0].hexsha, detach=True)
ensure_clean(repo)
for commit in commits[1:]:
try:
logger.info(f"Merging commit: {commit}")
result = repo.git.merge(commit.hexsha)
logger.info(result)
ensure_clean(repo)
except BaseException as e:
# Handle merge conflicts according to predefined rules
# For now, we'll abort the merge and report failure
repo.git.merge('--abort')
error_message = f"Merge conflict when merging {commit}: {str(e)}"
logger.error(error_message)
raise BaseException(error_message)
commit_resulting = repo.head.commit
repo.git.checkout(commit_resulting.hexsha, detach=True)
ensure_clean(repo)
remote_branch_name = None
sorted_commits = sorted(commit_ids.keys(), key=lambda x: x.hexsha)
version = '-'.join([x.hexsha[0:8] for x in sorted_commits])
for ref in repo.refs:
if ref.is_remote() and ref.commit == repo.head.commit and not ref.name.endswith('/HEAD'):
logger.info(f"Merge commit {ref.commit} corresponds to branch {ref}")
remote_name, branch_name = ref.name.split('/', 1)
remote_branch_name = branch_name if remote_name == 'origin' else ref.name
break
if self.push:
auto_branch_hash = hashlib.sha1(''.join([x.hexsha for x in sorted_commits]).encode()).hexdigest()
auto_branch_name = f"{AUTO_BRANCH_PREFIX}{version}"
push_remote = self.push if isinstance(self.push, str) else "origin"
logger.info(f"Pushing {repo.head.commit.hexsha} -> {auto_branch_name}")
repo.git.push('-f', push_remote, f"HEAD:refs/heads/{auto_branch_name}")
remote_branch_name = auto_branch_name
return CheckoutAndMergeResult(
wd=repo_path,
remote_branch_name=remote_branch_name,
commit_hash=repo.head.commit.hexsha,
version=version
)
from typing import Protocol, Any
@runtime_checkable
class HasVersion(Protocol):
version: str
@dataclass
class GitUnmergeResult:
branches: List[Tuple[str, str]] # (branch_name, commit_hash) pairs
columns: Dict[str, Dict[str, str]] # column_name -> {branch_name: value}
class GitUnmerge(AbstractStep[GitUnmergeResult]):
wd: GitClone
def __init__(self, wd: GitClone,
check: AbstractStep[Mapping[str, HasVersion]],
**kwargs: Any):
super().__init__(**kwargs)
self.wd = wd
self.check = check
self.warnings: List[str] = []
def progress(self) -> GitUnmergeResult:
self.warnings = []
wd = self.wd.progress()
repo = git.Repo(wd)
deployState: Mapping[str, HasVersion] = self.check.progress()
versions = set([it.version for it in deployState.values()])
if len(versions) != 1:
raise BaseException(f"Expected exactly one version, got: {versions}")
version = list(versions)[0]
if not version:
return GitUnmergeResult(branches=[], columns={"Deployed": {}})
if version is None:
raise BaseException(f"Expected exactly one version, got: {versions}")
childs: Dict[Commit, List[Commit]] | None = None
if version.startswith(AUTO_BRANCH_PREFIX):
version_parts = version[len(AUTO_BRANCH_PREFIX):].split('-')
commits: List[Tuple[str, str]] = []
for v in version_parts:
commit = repo.commit(v)
branches = [ref.name for ref in repo.refs
if ref.is_remote() and not ref.name.endswith('/HEAD') and ref.commit.hexsha == commit.hexsha]
branches = [b[len('origin/'):] if b.startswith('origin/') else b for b in branches]
if len(branches) == 0 or branches[0].startswith('auto/'):
if childs is None:
childs = _commits_childs(repo)
commitsSet = {commit}
while len(commitsSet) > 0:
current = commitsSet.pop()
for child in childs.get(current, []):
branches = [ref.name for ref in repo.refs
if ref.is_remote() and not ref.name.endswith('/HEAD') and ref.commit.hexsha == child.hexsha]
branches = [b[len('origin/'):] if b.startswith('origin/') else b for b in branches]
if len(branches) > 0:
commitsSet = set()
else:
commitsSet.add(child)
for b in branches:
commits.append((b, commit.hexsha))
if len(commits) == 0:
raise BaseException(f"Unable to unmerge version: {version}")
deployed_col: Dict[str, str] = {b: c[:8] for b, c in commits}
return GitUnmergeResult(
branches=commits,
columns={"Deployed": deployed_col},
)
else:
raise BaseException(f"Version format not recognized: {version}")