Merge main to all / rebase (push) Failing after 31s
Python package / build (3.12.3) (push) Failing after 2m28s
Python package / build (3.13) (push) Failing after 2m29s
Python package / build (3.14) (push) Failing after 2m38s
Python package / build (3.12.3) (pull_request) Failing after 2m17s
Python package / build (3.13) (pull_request) Canceled after 55s
Python package / build (3.14) (pull_request) Canceled after 0s
## Summary
`GitClone`'s top-level `branchNamePrefix` argument was silently ignored when constructed with the explicit `remotes=[...]` form (verified in `backend/steps/git.py:106`):
```python
if remote_name == "origin" and self.branchNamePrefix != "":
# apply refspec
```
The prefix only fires when a remote is named `"origin"`. The `remotes=` form passes names through verbatim, so configs like the open PR #28 set the prefix but never apply it — all branches from all remotes are fetched.
This PR extends the API: each remote entry now optionally accepts a `prefix` field. When set, the fetch refspec for that remote becomes scoped to `<prefix>/*` branches. Backwards-compatible — remotes without `prefix` keep the existing behavior, and `url=` shorthand with `branchNamePrefix` still works.
## API
```python
clone = GitClone(remotes=[
{"name": "github", "url": "...", "prefix": "ansible"},
{"name": "gitea_hermes", "url": "...", "prefix": "ansible"},
{"name": "gitea_admin", "url": "...", "prefix": "ansible"},
])
```
Each remote is independent — some can have `prefix`, others not.
## Changes
### `backend/steps/git.py`
- `__init__` (`remotes=` branch): validate `prefix` (string if set, reject non-string with `ValueError`), persist into the normalized entry. Empty string / None → no prefix applied (existing behavior).
- `progress()` fetch loop: replace the single `if remote_name == "origin"` check with per-entry logic. Effective prefix = `entry["prefix"] or (self.branchNamePrefix if remote_name == "origin" else "")`. Apply refspec per remote when effective prefix is non-empty.
### `tests/test_git_integration.py`
3 new tests in `TestGitIntegration`:
- `test_git_clone_remotes_per_remote_prefix` — both remotes prefixed; only `ansible/*` branches visible, `master` is filtered out.
- `test_git_clone_remotes_mixed_prefix` — one remote prefixed, one not; each behaves independently.
- `test_git_clone_remotes_prefix_validation` — `prefix=123` / `prefix=["ansible"]` raise `ValueError`; `prefix=""` and missing-`prefix` are accepted.
Existing `test_git_clone_named_remotes` continues to pass (verified — no behavior change for unprefixed remotes).
## Test results
```
29 passed in ~13s
```
(18 in `test_git_integration.py`, 11 in `test_dummy.py`/`test_caching_step.py`/`test_cached_env.py`.)
## Migration path
After merge:
- Open migration PRs (#26 gmail_mcp, #28 hermes, planned #19 gitea.py) — switch from `branchNamePrefix="ansible"` to `"prefix": "ansible"` on each remote.
- `hermes_infra.py` after PR #8 already drops `branchNamePrefix`, no change needed.
- Existing `url=` configs (immich, gmail_mcp-old, hermes-old, registry, gitea-old) — unaffected.
## Unblocks
PR #28 — once this lands, hermes.py can drop the (currently misleading) top-level `branchNamePrefix="ansible"` and add `"prefix": "ansible"` per remote.
Refs #30
Blocks #28
Part of #18 (parent migration ticket)
---------
Co-authored-by: admin <rudolfss13@gmail.com>
Co-authored-by: brencher_bot <rudolfss13@gmail.com>
Reviewed-on: http://192.168.1.96/admin/brencher/pulls/31
Co-authored-by: hermes <rudolfss13+hermes@gmail.com>
407 lines
14 KiB
Python
407 lines
14 KiB
Python
import hashlib
|
|
import logging
|
|
import os
|
|
import shutil
|
|
import tempfile
|
|
from dataclasses import dataclass
|
|
import traceback
|
|
from typing import List, Tuple, Set, Dict, Any, Mapping, runtime_checkable
|
|
|
|
import git
|
|
from enironment import AbstractStep, SharedState
|
|
from git.objects import Commit
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class GitClone(AbstractStep[str]):
|
|
def __init__(self,
|
|
url: str | List[str] | None = None,
|
|
remotes: List[Dict[str, str]] | None = None,
|
|
repo_path: str | None = None,
|
|
branchNamePrefix: str = "",
|
|
credEnvPrefix: str = "GIT",
|
|
**kwargs: Any):
|
|
super().__init__(**kwargs)
|
|
# `remotes` is the explicit form: a list of {"name": "...", "url": "..."}.
|
|
# `url` is kept as a shorthand: a single string becomes one remote named
|
|
# "origin"; a list of strings becomes remotes named "origin", "repo1",
|
|
# "repo2", ... (matches the behavior introduced by the "multiple remotes"
|
|
# change so existing positional-list callers keep working).
|
|
if remotes is not None and url is not None:
|
|
raise ValueError("Pass either 'url' or 'remotes', not both")
|
|
|
|
normalized: List[Dict[str, str]] = []
|
|
if remotes is not None:
|
|
seen: Set[str] = set()
|
|
for entry in remotes:
|
|
if not isinstance(entry, dict):
|
|
raise TypeError(
|
|
f"Each entry in 'remotes' must be a dict with 'name' and 'url', got {type(entry).__name__}"
|
|
)
|
|
name = entry.get("name")
|
|
url_value = entry.get("url")
|
|
if not name or not isinstance(name, str):
|
|
raise ValueError("Each remote requires a non-empty 'name' (string)")
|
|
if not url_value or not isinstance(url_value, str):
|
|
raise ValueError(f"Remote {name!r} requires a non-empty 'url' (string)")
|
|
if name in seen:
|
|
raise ValueError(f"Duplicate remote name: {name!r}")
|
|
prefix_value = entry.get("prefix")
|
|
if prefix_value is not None and not isinstance(prefix_value, str):
|
|
raise ValueError(
|
|
f"Remote {name!r} 'prefix' must be a string when set, got {type(prefix_value).__name__}"
|
|
)
|
|
seen.add(name)
|
|
normalized.append({"name": name, "url": url_value, "prefix": prefix_value or ""})
|
|
elif url is not None:
|
|
if isinstance(url, str):
|
|
raw = [url]
|
|
else:
|
|
raw = list(url)
|
|
for index, repository_url in enumerate(raw):
|
|
name = "origin" if index == 0 else f"repo{index}"
|
|
normalized.append({"name": name, "url": repository_url})
|
|
else:
|
|
raise ValueError("GitClone requires either 'url' or 'remotes'")
|
|
|
|
if len(normalized) == 0:
|
|
raise ValueError("At least one repository URL is required")
|
|
|
|
self.remotes = normalized
|
|
# Derived view used by repo_path keying and any introspection.
|
|
self.urls = [r["url"] for r in normalized]
|
|
self.repo_path = repo_path
|
|
self.branchNamePrefix = branchNamePrefix
|
|
self.credEnvPrefix = credEnvPrefix
|
|
|
|
def _get_auth_git_url(self, url: str) -> str:
|
|
username = os.getenv(f'{self.credEnvPrefix}_USERNAME')
|
|
password = os.getenv(f'{self.credEnvPrefix}_PASSWORD')
|
|
if username and password and '://' in url:
|
|
# Extract protocol and the rest of the URL
|
|
protocol, rest = url.split('://')
|
|
return f"{protocol}://{username}:{password}@{rest}"
|
|
|
|
return url
|
|
|
|
def progress(self) -> str:
|
|
self.repo_path = self.repo_path or os.path.join(tempfile.gettempdir(),
|
|
f"{self.env.id}_{hashlib.sha1('|'.join(self.urls).encode()).hexdigest()[:5]}")
|
|
logger.info(f"Cloning repositories {self.remotes} to {self.repo_path}")
|
|
os.makedirs(self.repo_path, exist_ok=True)
|
|
try:
|
|
if os.path.exists(os.path.join(self.repo_path, ".git")):
|
|
logger.info(f"Repository already cloned at {self.repo_path}, fetching updates.")
|
|
repo = git.Repo(self.repo_path)
|
|
else:
|
|
repo = git.Repo.init(self.repo_path)
|
|
|
|
for entry in self.remotes:
|
|
remote_name = entry["name"]
|
|
repository_url = entry["url"]
|
|
auth_url = self._get_auth_git_url(repository_url)
|
|
if remote_name in repo.remotes:
|
|
remote = repo.remotes[remote_name]
|
|
if remote.url != auth_url:
|
|
remote.set_url(auth_url)
|
|
else:
|
|
remote = repo.create_remote(remote_name, auth_url)
|
|
prefix = entry.get("prefix", "")
|
|
effective_prefix = prefix or (self.branchNamePrefix if remote_name == "origin" else "")
|
|
if effective_prefix:
|
|
with repo.config_writer() as config:
|
|
config.set_value(f'remote "{remote_name}"', "fetch",
|
|
f"+refs/heads/{effective_prefix}/*:refs/remotes/{remote_name}/{effective_prefix}/*")
|
|
result = remote.fetch(prune=True)
|
|
if not result or any(fetch_info.flags & fetch_info.ERROR for fetch_info in result):
|
|
raise BaseException(f"Failed to fetch updates for {repository_url}")
|
|
if not os.path.exists(os.path.join(self.repo_path, ".git")):
|
|
raise BaseException(f"Failed to clone repositories {self.remotes} to {self.repo_path}")
|
|
except BaseException as e:
|
|
logger.error(f"Error during git clone/fetch, removing directory {self.repo_path}: {str(e)}")
|
|
shutil.rmtree(self.repo_path)
|
|
raise e
|
|
return self.repo_path
|
|
|
|
def get_branches(self) -> Dict[str, List[Any]]:
|
|
repo = git.Repo(self.repo_path)
|
|
result: Dict[str, List[Any]] = {}
|
|
for ref in repo.refs:
|
|
if not ref.is_remote() or ref.name.endswith('/HEAD'):
|
|
continue
|
|
remote_name, branch_name = ref.name.split('/', 1)
|
|
if branch_name.startswith('auto/'): # Skip auto branches
|
|
continue
|
|
branch_key = branch_name if remote_name == 'origin' else ref.name
|
|
result[branch_key] = []
|
|
cmt = [ref.commit]
|
|
for _ in range(10):
|
|
for commit in cmt:
|
|
result[branch_key].append({
|
|
'hexsha': commit.hexsha,
|
|
'author': commit.author.name,
|
|
'date': commit.committed_datetime.isoformat(),
|
|
'message': commit.message.strip()
|
|
})
|
|
|
|
cmt = [p for c in cmt for p in c.parents]
|
|
return result
|
|
|
|
|
|
@dataclass
|
|
class CheckoutAndMergeResult:
|
|
wd: str
|
|
remote_branch_name: str | None
|
|
commit_hash: str
|
|
version: str
|
|
|
|
|
|
def _commits_childs(repo: git.Repo) -> Dict[Commit, List[Commit]]:
|
|
childs: Dict[Commit, List[Commit]] = {}
|
|
for c in list(repo.iter_commits('--all')):
|
|
for p in c.parents:
|
|
childs.setdefault(p, []).append(c)
|
|
return childs
|
|
|
|
|
|
def ensure_clean(repo: git.Repo) -> None:
|
|
if repo.is_dirty() or len(repo.untracked_files) > 0:
|
|
raise BaseException(f"Changes in repo: U{repo.untracked_files}")
|
|
|
|
|
|
def _resolve_branch_ref(repo: git.Repo, branch_key: str, default_remote: str) -> str:
|
|
# Branch keys can be "branch", "origin/branch", "gitea_admin/branch", etc.
|
|
# If the key has a remote prefix that matches a known remote, use it
|
|
# verbatim; otherwise fall back to the primary configured remote so
|
|
# configs that don't name a remote `origin` still work.
|
|
if "/" in branch_key:
|
|
remote_name, _ = branch_key.split("/", 1)
|
|
if remote_name in repo.remotes:
|
|
return branch_key
|
|
return f"{default_remote}/{branch_key}"
|
|
|
|
|
|
class CheckoutMerged(AbstractStep[CheckoutAndMergeResult]):
|
|
wd: GitClone
|
|
|
|
def __init__(self, wd: GitClone,
|
|
desired_branches: AbstractStep[SharedState],
|
|
git_user_email: str,
|
|
git_user_name: str,
|
|
push: bool = True,
|
|
**kwargs: Any):
|
|
super().__init__(**kwargs)
|
|
self.wd = wd
|
|
self.desired_branches = desired_branches
|
|
self.git_user_email = git_user_email
|
|
self.git_user_name = git_user_name
|
|
self.push = push
|
|
|
|
def _find_merge_childs(self, tree: Dict[Commit, List[Commit]], commit: Commit) -> List[Commit]:
|
|
|
|
visited = [commit]
|
|
queue = [commit]
|
|
queue = [child for c in queue for child in tree.get(c, [])]
|
|
|
|
while queue:
|
|
current = queue.pop(0)
|
|
visited.append(current)
|
|
for child in tree.get(current, []):
|
|
if len(child.parents) != 1:
|
|
queue.append(child)
|
|
|
|
return visited
|
|
|
|
def find_desired_commits(self, repo: git.Repo, branches: List[Tuple[str, str]]) -> Dict[Commit, str]:
|
|
# Extract commit ids for the selected branches
|
|
commit_ids: Dict[Commit, str] = {}
|
|
# Default remote: prefer the first remote GitClone was configured with
|
|
# (so non-"origin" configs like github/gitea_admin work without the
|
|
# caller having to qualify every branch key).
|
|
default_remote = "origin"
|
|
if isinstance(getattr(self.wd, "remotes", None), list) and self.wd.remotes:
|
|
primary = self.wd.remotes[0]["name"]
|
|
if primary in repo.remotes:
|
|
default_remote = primary
|
|
for branch_pair in branches:
|
|
try:
|
|
branch_name, desired_commit = branch_pair
|
|
if desired_commit == 'HEAD':
|
|
commit = repo.commit(_resolve_branch_ref(repo, branch_name, default_remote))
|
|
else:
|
|
commit = repo.commit(desired_commit)
|
|
commit_ids[commit] = branch_name
|
|
except BaseException as e:
|
|
stack = traceback.format_exception(type(e), e, e.__traceback__)
|
|
logger.error(f"Error finding desired commits for branch {branch_pair}: {str(e)}\n{''.join(stack)}")
|
|
return commit_ids
|
|
|
|
def progress(self) -> CheckoutAndMergeResult:
|
|
|
|
repo_path = self.wd.progress()
|
|
|
|
# Open the repository
|
|
repo = git.Repo(repo_path)
|
|
# Set user email and name for the repo
|
|
with repo.config_writer() as cw:
|
|
cw.set_value("user", "email", self.git_user_email)
|
|
cw.set_value("user", "name", self.git_user_name)
|
|
|
|
desired = self.desired_branches.progress().branches
|
|
if len(desired) == 0:
|
|
raise BaseException(f"Empty branches set")
|
|
|
|
logger.info(f"Selected branches: {desired}")
|
|
|
|
commit_ids = self.find_desired_commits(repo, desired)
|
|
logger.info(f"Commit ids for branches: {commit_ids}")
|
|
|
|
childs = _commits_childs(repo)
|
|
|
|
def find_common_merge_commits() -> Set[Commit]:
|
|
merge_commit: list[tuple[Commit, list[Commit]]] = [(c, self._find_merge_childs(childs, c)) for c in
|
|
commit_ids.keys()]
|
|
legal_merge_commits = [set(l) for c, l in merge_commit]
|
|
result = legal_merge_commits[0]
|
|
for l in legal_merge_commits[1:]:
|
|
result = result.intersection(l)
|
|
return result
|
|
|
|
if len(commit_ids) == 1:
|
|
commit_resulting = list(commit_ids.keys())[0]
|
|
logger.info(f"Only one branch selected, using commit {commit_resulting.hexsha}")
|
|
else:
|
|
merge_commits = find_common_merge_commits() # TODO There bugs here
|
|
commit_resulting = None
|
|
if len(merge_commits) > 0:
|
|
commit_resulting = merge_commits.pop()
|
|
logger.info(f"Common commit found {merge_commits}")
|
|
|
|
if commit_resulting is None:
|
|
logger.info(f"Merging commits")
|
|
# Merge the rest of the branches
|
|
commits = list(commit_ids.keys())
|
|
repo.git.checkout(commits[0].hexsha, detach=True)
|
|
ensure_clean(repo)
|
|
for commit in commits[1:]:
|
|
try:
|
|
logger.info(f"Merging commit: {commit}")
|
|
result = repo.git.merge(commit.hexsha)
|
|
logger.info(result)
|
|
ensure_clean(repo)
|
|
except BaseException as e:
|
|
# Handle merge conflicts according to predefined rules
|
|
# For now, we'll abort the merge and report failure
|
|
repo.git.merge('--abort')
|
|
error_message = f"Merge conflict when merging {commit}: {str(e)}"
|
|
logger.error(error_message)
|
|
|
|
raise BaseException(error_message)
|
|
commit_resulting = repo.head.commit
|
|
|
|
repo.git.checkout(commit_resulting.hexsha, detach=True)
|
|
ensure_clean(repo)
|
|
remote_branch_name = None
|
|
sorted_commits = sorted(commit_ids.keys(), key=lambda x: x.hexsha)
|
|
version = '-'.join([x.hexsha[0:8] for x in sorted_commits])
|
|
|
|
for ref in repo.refs:
|
|
if ref.is_remote() and ref.commit == repo.head.commit and not ref.name.endswith('/HEAD'):
|
|
logger.info(f"Merge commit {ref.commit} corresponds to branch {ref}")
|
|
remote_name, branch_name = ref.name.split('/', 1)
|
|
remote_branch_name = branch_name if remote_name == 'origin' else ref.name
|
|
break
|
|
if self.push and remote_branch_name is None:
|
|
auto_branch_hash = hashlib.sha1(''.join([x.hexsha for x in sorted_commits]).encode()).hexdigest()
|
|
auto_branch_name = f"auto-{version}"
|
|
|
|
logger.info(f"Pushing {repo.head.commit.hexsha} -> {auto_branch_name}")
|
|
repo.git.push('-f', 'origin', f"HEAD:refs/heads/{auto_branch_name}")
|
|
remote_branch_name = auto_branch_name
|
|
|
|
return CheckoutAndMergeResult(
|
|
wd=repo_path,
|
|
remote_branch_name=remote_branch_name,
|
|
commit_hash=repo.head.commit.hexsha,
|
|
version=version
|
|
)
|
|
|
|
|
|
from typing import Protocol, Any
|
|
|
|
|
|
@runtime_checkable
|
|
class HasVersion(Protocol):
|
|
version: str
|
|
|
|
|
|
@dataclass
|
|
class GitUnmergeResult:
|
|
branches: List[Tuple[str, str]] # (branch_name, commit_hash) pairs
|
|
columns: Dict[str, Dict[str, str]] # column_name -> {branch_name: value}
|
|
|
|
|
|
class GitUnmerge(AbstractStep[GitUnmergeResult]):
|
|
wd: GitClone
|
|
|
|
def __init__(self, wd: GitClone,
|
|
check: AbstractStep[Mapping[str, HasVersion]],
|
|
**kwargs: Any):
|
|
super().__init__(**kwargs)
|
|
self.wd = wd
|
|
self.check = check
|
|
|
|
def progress(self) -> GitUnmergeResult:
|
|
wd = self.wd.progress()
|
|
deployState: Mapping[str, HasVersion] = self.check.progress()
|
|
|
|
versions = set([it.version for it in deployState.values()])
|
|
if len(versions) != 1:
|
|
raise BaseException(f"Expected exactly one version, got: {versions}")
|
|
|
|
version = list(versions)[0]
|
|
if version is None:
|
|
raise BaseException(f"Expected exactly one version, got: {versions}")
|
|
repo = git.Repo(wd)
|
|
childs: Dict[Commit, List[Commit]] | None = None
|
|
if 'auto-' in version:
|
|
version_parts = version[len('auto-'):].split('-')
|
|
|
|
commits: List[Tuple[str, str]] = []
|
|
for v in version_parts:
|
|
commit = repo.commit(v)
|
|
branches = [ref.name for ref in repo.refs
|
|
if ref.is_remote() and not ref.name.endswith('/HEAD') and ref.commit.hexsha == commit.hexsha]
|
|
branches = [b[len('origin/'):] if b.startswith('origin/') else b for b in branches]
|
|
|
|
if len(branches) == 0 or branches[0].startswith('auto/'):
|
|
if childs is None:
|
|
childs = _commits_childs(repo)
|
|
commitsSet = {commit}
|
|
while len(commitsSet) > 0:
|
|
current = commitsSet.pop()
|
|
for child in childs.get(current, []):
|
|
branches = [ref.name for ref in repo.refs
|
|
if ref.is_remote() and not ref.name.endswith('/HEAD') and ref.commit.hexsha == child.hexsha]
|
|
branches = [b[len('origin/'):] if b.startswith('origin/') else b for b in branches]
|
|
|
|
if len(branches) > 0:
|
|
commitsSet = set()
|
|
else:
|
|
commitsSet.add(child)
|
|
for b in branches:
|
|
commits.append((b, commit.hexsha))
|
|
|
|
if len(commits) == 0:
|
|
raise BaseException(f"Unable to unmerge version: {version}")
|
|
|
|
deployed_col: Dict[str, str] = {b: c[:8] for b, c in commits}
|
|
return GitUnmergeResult(
|
|
branches=commits,
|
|
columns={"Deployed": deployed_col},
|
|
)
|
|
else:
|
|
raise BaseException(f"Version format not recognized: {version}")
|