Compare commits

..
14 changed files with 101 additions and 506 deletions
+34 -35
View File
@@ -46,38 +46,37 @@ jobs:
- name: Run checker
run: ${{ matrix.checker }}
# tests:
# runs-on: ubuntu-latest
# needs: build
# strategy:
# fail-fast: false
# matrix:
# python-version: [
# # "3.11.3",
# "3.12.3",
# "3.13",
# "3.14",
# # "3.15",
# ]
# checker: [
# "pytest",
# ]
# steps:
# - uses: actions/checkout@v4
# - name: Set up Python ${{ matrix.python-version }}, ${{ matrix.checker }}
# uses: actions/setup-python@v3
# with:
# python-version: ${{ matrix.python-version }}
# - name: Install dependencies
# run: |
# # sudo apt-get update
# # sudo apt-get upgrade -y
# # sudo apt-get install -y --no-install-recommends git docker.io
# python -m pip install --upgrade pip
# pip install -r requirements-dev.txt
# playwright install --with-deps chromium
# - name: Run checker
# run: "pytest"
#
#
#
tests:
runs-on: ubuntu-latest
needs: build
strategy:
fail-fast: false
matrix:
python-version: [
# "3.11.3",
"3.12.3",
"3.13",
"3.14",
# "3.15",
]
checker: [
"pytest",
]
steps:
- uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}, ${{ matrix.checker }}
uses: actions/setup-python@v3
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: |
# sudo apt-get update
# sudo apt-get upgrade -y
# sudo apt-get install -y --no-install-recommends git docker.io
python -m pip install --upgrade pip
pip install -r requirements-dev.txt
playwright install --with-deps chromium
- name: Run checker
run: "pytest"
+2 -2
View File
@@ -48,7 +48,7 @@ class App:
"name": r.name,
"status": [str(result), stack],
"error": True,
"is_running": False,
"is_running": True,
})
else:
pipeline_state.append({
@@ -62,7 +62,7 @@ class App:
"name": r.name,
"status": [str(e), stack],
"error": True,
"is_running": False,
"is_running": True,
})
env_dtos[env.id] = {'id': env.id }
env_dtos[env.id]['pipeline'] = pipeline_state
-61
View File
@@ -1,61 +0,0 @@
from steps.git import GitClone, CheckoutMerged, GitUnmerge
from steps.docker import DockerSwarmCheck, DockerSwarmDeploy
from enironment import Environment
from steps.checks import SimpleLog, UrlCheck
from steps.shared_state import SharedStateHolderInMemory
clone = GitClone(url="https://github.com/rudolf1/uber_backup.git", branchNamePrefix="photosHelper")
sharedState = SharedStateHolderInMemory(unmerge=None)
checkoutMerged = CheckoutMerged(clone,
desired_branches=sharedState,
push = False,
git_user_email="rudolfss13@gmail.com",
git_user_name="brencher_bot"
)
sharedState.set_branches([("photosHelper/master", "HEAD")])
dockerSwarmCheck = DockerSwarmCheck(
stack_name = "photosHelper",
)
deployDocker = DockerSwarmDeploy(
wd=clone,
buildDocker=None,
stackChecker=dockerSwarmCheck,
envs = lambda: {
"version": "auto-" + checkoutMerged.progress().version
},
stack_name = "photosHelper",
docker_compose_path = "docker-compose.yml",
)
unmerge = GitUnmerge(clone, dockerSwarmCheck)
checkPing = UrlCheck(
url="https://backuper.rudolf.keenetic.link/api/server/ping",
expected = {"res":"pong"},
)
logUrls = SimpleLog(message = {
"userLinks": {
"App": "https://backuper.rudolf.keenetic.link",
}
})
backups_helper = Environment(
id="backupsHelper",
state=sharedState,
pipeline=[
clone,
sharedState,
checkoutMerged,
dockerSwarmCheck,
unmerge,
deployDocker,
checkPing,
logUrls
]
)
+1 -5
View File
@@ -5,11 +5,7 @@ from steps.docker import DockerComposeBuild, DockerSwarmCheck, DockerSwarmDeploy
from steps.git import GitClone, CheckoutMerged, GitUnmerge
from steps.shared_state import SharedStateHolderInMemory
clone = GitClone(remotes=[
{"name": "github", "url": "https://github.com/rudolf1/brencher.git"},
{"name": "gitea_admin", "url": "https://gitea.rudolf.keenetic.link/admin/brencher"},
{"name": "gitea_hermes", "url": "https://gitea.rudolf.keenetic.link/hermes/brencher"},
])
clone = GitClone(url="https://github.com/rudolf1/brencher.git")
dockerSwarmCheck = DockerSwarmCheck(
stack_name="brencher",
+1 -5
View File
@@ -21,11 +21,7 @@ def checkPingF(obj: Any) -> None:
raise Exception(f"brencher2 check failed for: {v}")
clone = GitClone(remotes=[
{"name": "github", "url": "https://github.com/rudolf1/brencher.git"},
{"name": "gitea_admin", "url": "https://gitea.rudolf.keenetic.link/admin/brencher"},
{"name": "gitea_hermes", "url": "https://gitea.rudolf.keenetic.link/hermes/brencher"},
])
clone = GitClone(url="https://github.com/rudolf1/brencher.git")
dockerSwarmCheck = DockerSwarmCheck(
stack_name="brencher2",
+1 -22
View File
@@ -4,28 +4,7 @@ from steps.docker import DockerComposeBuild, DockerSwarmCheck, DockerSwarmDeploy
from steps.git import GitClone, CheckoutMerged, GitUnmerge
from steps.shared_state import SharedStateHolderInMemory
# Dual-remote clone for the ansible mirror (the deploy target, not brencher itself).
# - gitea_hermes is where the feature branch lives (hermes/ansible:*)
# - gitea_admin is the live-deploy source (where brencher-bot reads compose files from)
#
# Why not github? github.com is unreachable from the brencher container -- the
# proxy returns HTTP 403 on CONNECT tunnel, so the URL would always fail to
# fetch. See backend/configs/docker_socket_proxy.py for the same migration.
#
# Why drop branchNamePrefix? GitClone only applies its prefix refspec to a
# remote named "origin" (backend/steps/git.py ~L104); neither remote here is
# named "origin", so the prefix would be a silent no-op. Both remotes fetch
# their full refset instead.
#
# Why two remotes? The deploy step (DockerSwarmDeploy) checks out a specific
# branch and uses the resulting checkout as wd for the compose path. With
# gitea_hermes as primary, feature branches opened in the fork are picked up;
# gitea_admin provides the live-deploy source as a fallback / for cross-fork
# PRs. This matches the form in backend/configs/brencher.py after PR #3.
clone = GitClone(remotes=[
{"name": "gitea_hermes", "url": "https://gitea.rudolf.keenetic.link/hermes/ansible"},
{"name": "gitea_admin", "url": "https://gitea.rudolf.keenetic.link/admin/ansible"},
])
clone = GitClone(url="https://github.com/rudolf1/uber_backup.git", branchNamePrefix="ansible")
dockerSwarmCheck = DockerSwarmCheck(
stack_name="hermes_infra",
+1 -5
View File
@@ -3,11 +3,7 @@ from steps.docker import DockerComposeBuild, DockerSwarmCheck, DockerSwarmDeploy
from steps.git import GitClone, CheckoutMerged, GitUnmerge
from steps.shared_state import SharedStateHolderInMemory
clone = GitClone(remotes=[
{"name": "github", "url": "https://github.com/rudolf1/reverseproxy.git"},
{"name": "gitea_admin", "url": "https://gitea.rudolf.keenetic.link/admin/reverseproxy"},
{"name": "gitea_hermes", "url": "https://gitea.rudolf.keenetic.link/hermes/reverseproxy"},
])
clone = GitClone(url="https://github.com/rudolf1/reverseproxy.git")
dockerSwarmCheck = DockerSwarmCheck(
stack_name="torrserv_proxy",
+4 -68
View File
@@ -1,4 +1,3 @@
import hashlib
import logging
import os
import re
@@ -100,7 +99,7 @@ class DockerComposeBuild(AbstractStep[Dict[str, str]]):
pass
logger.info(f"Building image {image} from {build_ctx}, {build_dockerfile}")
img, _ = client.images.build(path=build_ctx, dockerfile=build_dockerfile, tag=image, nocache=not self.build_cache, rm=not self.build_cache)
img, _ = client.images.build(path=build_ctx, dockerfile=build_dockerfile, tag=image, nocache=not self.build_cache, rm=True)
if self.publish:
logger.info(f"Pushing image {image}")
@@ -198,32 +197,6 @@ class DockerSwarmDeploy(AbstractStep[str]):
content = f.read()
content = re.sub(r'\$\{([^}]+)\}', lambda m: env.get(m.group(1), ""), content)
compose = yaml.safe_load(content)
version = env["version"]
if "configs" in compose:
sanitized_version = re.sub(r'[^a-zA-Z0-9_.-]', '_', version)
full_prefix = f"{self.stack_name}_"
def _config_full_name(base_name: str) -> str:
full = f"{full_prefix}{base_name}_{sanitized_version}"
if len(full) <= 64:
return f"{base_name}_{sanitized_version}"
return hashlib.sha256(full.encode("utf-8")).hexdigest()[:32]
config_names = {
name: _config_full_name(name)
for name in compose["configs"]
}
compose["configs"] = {
config_names.get(name, name): config
for name, config in compose["configs"].items()
}
for service in compose.get("services", {}).values():
for config in service.get("configs", []):
if isinstance(config, str):
config_index = service["configs"].index(config)
service["configs"][config_index] = config_names.get(config, config)
elif isinstance(config, dict) and "source" in config:
config["source"] = config_names.get(config["source"], config["source"])
if "services" in compose:
for svc in compose["services"].values():
if "build" in svc:
@@ -295,7 +268,6 @@ class DockerSwarmDeploy(AbstractStep[str]):
# Deploy stack to Docker Swarm
cmd = [
"docker", "stack", "deploy",
"--detach=false",
"--prune",
"-c", tmp_compose_path,
self.stack_name
@@ -310,24 +282,9 @@ class DockerSwarmDeploy(AbstractStep[str]):
result = subprocess.run(cmd, capture_output=True, text=True, cwd=os.path.dirname(tmp_compose_path))
#, env=swarmEnv)
os.remove(tmp_compose_path)
logger.info(f"docker stack deploy stdout: {result.stdout or '<empty>'}")
logger.info(f"docker stack deploy stderr: {result.stderr or '<empty>'}")
stderr_lower = result.stderr.lower()
docker_error_markers = (
"failed to ",
"error response from daemon",
"rpc error",
"denied",
"no such image",
)
if result.returncode != 0 or any(marker in stderr_lower for marker in docker_error_markers):
deploy_output = (
f"exit code: {result.returncode}\n"
f"stdout:\n{result.stdout or '<empty>'}\n"
f"stderr:\n{result.stderr or '<empty>'}"
)
logger.error(f"Stack deploy failed:\n{deploy_output}")
raise RuntimeError(f"Stack deploy failed:\n{deploy_output}")
if result.returncode != 0:
logger.error(f"Stack deploy failed: {result.stderr}")
raise RuntimeError(f"Stack deploy failed: {result.stderr}")
logger.info(f"Stack deployed successfully: {result.stdout}")
# Check health of all deployed services
@@ -351,25 +308,4 @@ class DockerSwarmDeploy(AbstractStep[str]):
if not_running:
raise BaseException(f"Services not started yet: {', '.join(not_running)}")
version_mismatches = []
for svc_name, svc in expected_services.items():
full_svc_name = f"{self.stack_name}_{svc_name}"
services_list = client.services.list(filters={"name": full_svc_name})
if not services_list:
continue
service_labels = services_list[0].attrs.get("Spec", {}).get("TaskTemplate", {}).get("ContainerSpec", {}).get("Labels", {})
expected_version = svc.get("labels", {}).get("org.brencher.version")
actual_version = service_labels.get("org.brencher.version")
if expected_version != actual_version:
version_mismatches.append(
f"{full_svc_name} expected version {expected_version}, got {actual_version}"
)
update_status = services_list[0].attrs.get("UpdateStatus", {})
if update_status.get("State") in {"paused", "rollback_paused", "rollback_started"}:
version_mismatches.append(
f"{full_svc_name} Swarm update {update_status.get('State')}: {update_status.get('Message', '')}"
)
if version_mismatches:
raise RuntimeError(f"Deployment did not apply requested versions: {'; '.join(version_mismatches)}")
return f"Stack deployed successfully: {result.stdout}"
+48 -122
View File
@@ -16,57 +16,11 @@ logger = logging.getLogger(__name__)
class GitClone(AbstractStep[str]):
def __init__(self,
url: str | List[str] | None = None,
remotes: List[Dict[str, str]] | None = None,
repo_path: str | None = None,
branchNamePrefix: str = "",
credEnvPrefix: str = "GIT",
def __init__(self, url: str, repo_path: str | None = None,
branchNamePrefix: str = "", credEnvPrefix: str = "GIT",
**kwargs: Any):
super().__init__(**kwargs)
# `remotes` is the explicit form: a list of {"name": "...", "url": "..."}.
# `url` is kept as a shorthand: a single string becomes one remote named
# "origin"; a list of strings becomes remotes named "origin", "repo1",
# "repo2", ... (matches the behavior introduced by the "multiple remotes"
# change so existing positional-list callers keep working).
if remotes is not None and url is not None:
raise ValueError("Pass either 'url' or 'remotes', not both")
normalized: List[Dict[str, str]] = []
if remotes is not None:
seen: Set[str] = set()
for entry in remotes:
if not isinstance(entry, dict):
raise TypeError(
f"Each entry in 'remotes' must be a dict with 'name' and 'url', got {type(entry).__name__}"
)
name = entry.get("name")
url_value = entry.get("url")
if not name or not isinstance(name, str):
raise ValueError("Each remote requires a non-empty 'name' (string)")
if not url_value or not isinstance(url_value, str):
raise ValueError(f"Remote {name!r} requires a non-empty 'url' (string)")
if name in seen:
raise ValueError(f"Duplicate remote name: {name!r}")
seen.add(name)
normalized.append({"name": name, "url": url_value})
elif url is not None:
if isinstance(url, str):
raw = [url]
else:
raw = list(url)
for index, repository_url in enumerate(raw):
name = "origin" if index == 0 else f"repo{index}"
normalized.append({"name": name, "url": repository_url})
else:
raise ValueError("GitClone requires either 'url' or 'remotes'")
if len(normalized) == 0:
raise ValueError("At least one repository URL is required")
self.remotes = normalized
# Derived view used by repo_path keying and any introspection.
self.urls = [r["url"] for r in normalized]
self.url = url
self.repo_path = repo_path
self.branchNamePrefix = branchNamePrefix
self.credEnvPrefix = credEnvPrefix
@@ -82,36 +36,33 @@ class GitClone(AbstractStep[str]):
return url
def progress(self) -> str:
repo_url = self.url
self.repo_path = self.repo_path or os.path.join(tempfile.gettempdir(),
f"{self.env.id}_{hashlib.sha1('|'.join(self.urls).encode()).hexdigest()[:5]}")
logger.info(f"Cloning repositories {self.remotes} to {self.repo_path}")
f"{self.env.id}_{hashlib.sha1(repo_url.encode()).hexdigest()[:5]}")
logger.info(f"Cloning repository {repo_url} to {self.repo_path}")
os.makedirs(self.repo_path, exist_ok=True)
try:
if os.path.exists(os.path.join(self.repo_path, ".git")):
logger.info(f"Repository already cloned at {self.repo_path}, fetching updates.")
repo = git.Repo(self.repo_path)
if 'origin' not in repo.remotes or repo.remotes.origin.url != self._get_auth_git_url(repo_url):
repo.create_remote('origin', self._get_auth_git_url(repo_url))
result = repo.remotes.origin.fetch(prune=True)
if not result or any(fetch_info.flags & fetch_info.ERROR for fetch_info in result):
raise BaseException(f"Failed to fetch updates for {repo_url}")
else:
repo = git.Repo.init(self.repo_path)
for entry in self.remotes:
remote_name = entry["name"]
repository_url = entry["url"]
auth_url = self._get_auth_git_url(repository_url)
if remote_name in repo.remotes:
remote = repo.remotes[remote_name]
if remote.url != auth_url:
remote.set_url(auth_url)
else:
remote = repo.create_remote(remote_name, auth_url)
if remote_name == "origin" and self.branchNamePrefix != "":
with repo.config_writer() as config:
config.set_value('remote "origin"', "fetch",
f"+refs/heads/{self.branchNamePrefix}/*:refs/remotes/origin/{self.branchNamePrefix}/*")
result = remote.fetch(prune=True)
if not result or any(fetch_info.flags & fetch_info.ERROR for fetch_info in result):
raise BaseException(f"Failed to fetch updates for {repository_url}")
if not os.path.exists(os.path.join(self.repo_path, ".git")):
raise BaseException(f"Failed to clone repositories {self.remotes} to {self.repo_path}")
repo.remotes.append(repo.create_remote(
'origin',
self._get_auth_git_url(repo_url)
))
if self.branchNamePrefix != "":
repo.config_writer().set_value('remote "origin"', "fetch",
f"+refs/heads/{self.branchNamePrefix}/*:refs/remotes/origin/{self.branchNamePrefix}/*").release()
repo.remotes.origin.fetch(prune=True)
if not os.path.exists(os.path.join(self.repo_path, ".git")):
raise BaseException(f"Failed to clone repository {repo_url} to {self.repo_path}")
except BaseException as e:
logger.error(f"Error during git clone/fetch, removing directory {self.repo_path}: {str(e)}")
shutil.rmtree(self.repo_path)
@@ -122,24 +73,21 @@ class GitClone(AbstractStep[str]):
repo = git.Repo(self.repo_path)
result: Dict[str, List[Any]] = {}
for ref in repo.refs:
if not ref.is_remote() or ref.name.endswith('/HEAD'):
continue
remote_name, branch_name = ref.name.split('/', 1)
if branch_name.startswith('auto/'): # Skip auto branches
continue
branch_key = branch_name if remote_name == 'origin' else ref.name
result[branch_key] = []
cmt = [ref.commit]
for _ in range(10):
for commit in cmt:
result[branch_key].append({
'hexsha': commit.hexsha,
'author': commit.author.name,
'date': commit.committed_datetime.isoformat(),
'message': commit.message.strip()
})
if ref.name.startswith('origin/') and not ref.name.startswith('origin/HEAD'):
branch_name = ref.name[len('origin/'):]
if not branch_name.startswith('auto/'): # Skip auto branches
result[branch_name] = []
cmt = [ref.commit]
for _ in range(10):
for commit in cmt:
result[branch_name].append({
'hexsha': commit.hexsha,
'author': commit.author.name,
'date': commit.committed_datetime.isoformat(),
'message': commit.message.strip()
})
cmt = [p for c in cmt for p in c.parents]
cmt = [p for c in cmt for p in c.parents]
return result
@@ -164,18 +112,6 @@ def ensure_clean(repo: git.Repo) -> None:
raise BaseException(f"Changes in repo: U{repo.untracked_files}")
def _resolve_branch_ref(repo: git.Repo, branch_key: str, default_remote: str) -> str:
# Branch keys can be "branch", "origin/branch", "gitea_admin/branch", etc.
# If the key has a remote prefix that matches a known remote, use it
# verbatim; otherwise fall back to the primary configured remote so
# configs that don't name a remote `origin` still work.
if "/" in branch_key:
remote_name, _ = branch_key.split("/", 1)
if remote_name in repo.remotes:
return branch_key
return f"{default_remote}/{branch_key}"
class CheckoutMerged(AbstractStep[CheckoutAndMergeResult]):
wd: GitClone
@@ -183,7 +119,6 @@ class CheckoutMerged(AbstractStep[CheckoutAndMergeResult]):
desired_branches: AbstractStep[SharedState],
git_user_email: str,
git_user_name: str,
ignore_missing_branches: bool = True,
push: bool = True,
**kwargs: Any):
super().__init__(**kwargs)
@@ -192,7 +127,7 @@ class CheckoutMerged(AbstractStep[CheckoutAndMergeResult]):
self.git_user_email = git_user_email
self.git_user_name = git_user_name
self.push = push
self.ignore_missing_branches = ignore_missing_branches
def _find_merge_childs(self, tree: Dict[Commit, List[Commit]], commit: Commit) -> List[Commit]:
visited = [commit]
@@ -211,25 +146,15 @@ class CheckoutMerged(AbstractStep[CheckoutAndMergeResult]):
def find_desired_commits(self, repo: git.Repo, branches: List[Tuple[str, str]]) -> Dict[Commit, str]:
# Extract commit ids for the selected branches
commit_ids: Dict[Commit, str] = {}
# Default remote: prefer the first remote GitClone was configured with
# (so non-"origin" configs like github/gitea_admin work without the
# caller having to qualify every branch key).
default_remote = "origin"
if isinstance(getattr(self.wd, "remotes", None), list) and self.wd.remotes:
primary = self.wd.remotes[0]["name"]
if primary in repo.remotes:
default_remote = primary
for branch_pair in branches:
try:
branch_name, desired_commit = branch_pair
if desired_commit == 'HEAD':
commit = repo.commit(_resolve_branch_ref(repo, branch_name, default_remote))
commit = repo.commit(f'origin/{branch_name}')
else:
commit = repo.commit(desired_commit)
commit_ids[commit] = branch_name
except BaseException as e:
if not self.ignore_missing_branches:
raise e
stack = traceback.format_exception(type(e), e, e.__traceback__)
logger.error(f"Error finding desired commits for branch {branch_pair}: {str(e)}\n{''.join(stack)}")
return commit_ids
@@ -304,10 +229,9 @@ class CheckoutMerged(AbstractStep[CheckoutAndMergeResult]):
version = '-'.join([x.hexsha[0:8] for x in sorted_commits])
for ref in repo.refs:
if ref.is_remote() and ref.commit == repo.head.commit and not ref.name.endswith('/HEAD'):
if ref.is_remote() and ref.commit == repo.head.commit and ref.name != 'origin/HEAD':
logger.info(f"Merge commit {ref.commit} corresponds to branch {ref}")
remote_name, branch_name = ref.name.split('/', 1)
remote_branch_name = branch_name if remote_name == 'origin' else ref.name
remote_branch_name = ref.name[len('origin/'):]
break
if self.push and remote_branch_name is None:
auto_branch_hash = hashlib.sha1(''.join([x.hexsha for x in sorted_commits]).encode()).hexdigest()
@@ -368,9 +292,10 @@ class GitUnmerge(AbstractStep[GitUnmergeResult]):
commits: List[Tuple[str, str]] = []
for v in version_parts:
commit = repo.commit(v)
branches = [ref.name for ref in repo.refs
if ref.is_remote() and not ref.name.endswith('/HEAD') and ref.commit.hexsha == commit.hexsha]
branches = [b[len('origin/'):] if b.startswith('origin/') else b for b in branches]
# branches = [head.name for head in repo.heads if head.commit.hexsha == commit.hexsha]
branches = [ref.name for ref in repo.remotes.origin.refs if ref.commit.hexsha == commit.hexsha]
branches = [b[len('origin/'):] for b in branches if
b.startswith('origin/') and not b.startswith('origin/HEAD')]
if len(branches) == 0 or branches[0].startswith('auto/'):
if childs is None:
@@ -379,9 +304,10 @@ class GitUnmerge(AbstractStep[GitUnmergeResult]):
while len(commitsSet) > 0:
current = commitsSet.pop()
for child in childs.get(current, []):
branches = [ref.name for ref in repo.refs
if ref.is_remote() and not ref.name.endswith('/HEAD') and ref.commit.hexsha == child.hexsha]
branches = [b[len('origin/'):] if b.startswith('origin/') else b for b in branches]
branches = [ref.name for ref in repo.remotes.origin.refs if
ref.commit.hexsha == child.hexsha]
branches = [b[len('origin/'):] for b in branches if
b.startswith('origin/') and not b.startswith('origin/HEAD')]
if len(branches) > 0:
commitsSet = set()
-7
View File
@@ -60,10 +60,3 @@ class CachingStep(AbstractStep[T], Generic[T]):
def __getattr__(self, name: str) -> Any:
"""Delegate unknown attributes to the wrapped step."""
return getattr(self._step, name)
def __setattr__(self, name: str, value: Any) -> None:
"""Delegate unknown attributes to the wrapped step."""
if name in {"name", "_step", "_result", "_input_hash"}:
super().__setattr__(name, value)
else:
setattr(self._step, name, value)
+1
View File
@@ -0,0 +1 @@
TODO
+1 -86
View File
@@ -5,12 +5,10 @@ These tests run in a Docker-like isolated environment to avoid harming the host
They simulate remote and local git repositories and test various merge scenarios.
"""
import os
import shutil
import tempfile
import git
import pytest
from steps.git import CheckoutAndMergeResult, GitClone, GitUnmergeResult
from steps.git import CheckoutAndMergeResult, GitUnmergeResult
from .test_remote_repo import RemoteRepoHelper
@@ -18,89 +16,6 @@ from .test_remote_repo import RemoteRepoHelper
class TestGitIntegration:
"""Integration tests for git operations"""
def test_git_clone_multiple_urls(self, repo_helper: RemoteRepoHelper) -> None:
repo_helper.create_commit(repo_helper.repo, "master", "master", "README.md", "content", "Initial commit")
second_url = f"file://{repo_helper.remote_dir}"
clone = GitClone(url=[repo_helper.remote_dir, second_url])
clone.env = repo_helper.env
repo_path = clone.progress()
repo = git.Repo(repo_path)
assert {remote.name for remote in repo.remotes} == {"origin", "repo1"}
assert repo.remotes.origin.url == repo_helper.remote_dir
assert repo.remotes.repo1.url == second_url
assert set(clone.get_branches()) == {"master", "repo1/master"}
def test_git_clone_named_remotes(self, repo_helper: RemoteRepoHelper) -> None:
# Two real local repos served as separate "remotes". The user controls
# their names explicitly via the `remotes=[{name,url}]` API.
def fresh_repo(prefix: str, message: str) -> str:
path = tempfile.mkdtemp(prefix=prefix)
repo = git.Repo.init(path, bare=False)
with repo.config_writer() as cw:
cw.set_value("user", "email", "test@example.com")
cw.set_value("user", "name", "Test User")
repo_helper.create_commit(repo, "master", "master", "README.md", "content", message)
return path
github_dir = fresh_repo("test_named_github_", "github initial")
gitea_dir = fresh_repo("test_named_gitea_", "gitea initial")
try:
clone = GitClone(remotes=[
{"name": "github", "url": github_dir},
{"name": "gitea_admin", "url": gitea_dir},
])
clone.env = repo_helper.env
repo_path = clone.progress()
repo = git.Repo(repo_path)
# User-chosen names appear exactly; the auto-generated "origin" /
# "repo1" names from the positional-list form do NOT leak in.
assert {r.name for r in repo.remotes} == {"github", "gitea_admin"}
assert "origin" not in repo.remotes
assert "repo1" not in repo.remotes
assert repo.remotes.github.url == github_dir
assert repo.remotes.gitea_admin.url == gitea_dir
# `get_branches` keeps the <remote>/<branch> shape for non-origin
# names; the user gets the full provenance in the key.
assert set(clone.get_branches()) == {"github/master", "gitea_admin/master"}
finally:
shutil.rmtree(github_dir, ignore_errors=True)
shutil.rmtree(gitea_dir, ignore_errors=True)
def test_git_clone_remotes_validation(self) -> None:
# Empty list: rejected (would produce an unusable clone).
with pytest.raises(ValueError, match="At least one repository URL is required"):
GitClone(remotes=[])
# Missing / empty / non-string name or url: rejected.
with pytest.raises(ValueError, match="non-empty 'name'"):
GitClone(remotes=[{"url": "git@example.com:foo/bar.git"}])
with pytest.raises(ValueError, match="non-empty 'name'"):
GitClone(remotes=[{"name": "", "url": "git@example.com:foo/bar.git"}])
with pytest.raises(ValueError, match="non-empty 'url'"):
GitClone(remotes=[{"name": "github"}])
with pytest.raises(ValueError, match="non-empty 'url'"):
GitClone(remotes=[{"name": "github", "url": ""}])
# Duplicate name: rejected.
with pytest.raises(ValueError, match="Duplicate remote name"):
GitClone(remotes=[
{"name": "github", "url": "git@example.com:a/b.git"},
{"name": "github", "url": "git@example.com:c/d.git"},
])
# Passing both `url` and `remotes`: rejected.
with pytest.raises(ValueError, match="not both"):
GitClone(url="git@example.com:a/b.git", remotes=[{"name": "github", "url": "git@example.com:a/b.git"}])
# Passing neither: rejected.
with pytest.raises(ValueError, match="requires either 'url' or 'remotes'"):
GitClone()
def test_checkout_merged_one_branch(self, repo_helper: RemoteRepoHelper) -> None:
# Create main branch with initial commit
commit1 = repo_helper.create_commit(repo_helper.repo, "master", "master", "file1.txt", "content1",
-61
View File
@@ -1,61 +0,0 @@
from git import BadName
import pytest
from enironment import get_step
from steps.git import CheckoutMerged
from tests.conftest import repo_helper
from .test_remote_repo import RemoteRepoHelper
class TestGitRemoveBranch:
"""Integration tests for git operations"""
def test_checkout_merged_one_branch(self, repo_helper: RemoteRepoHelper) -> None:
# Create main branch with initial commit
commit1 = repo_helper.create_commit(repo_helper.repo, "master", "master", "file1.txt", "content1",
"Initial commit") # noqa: F841
commit2 = repo_helper.create_commit(repo_helper.repo, "master", "branch1", "file2.txt", "content2",
"Branch1 commit") # noqa: F841
commit3 = repo_helper.create_commit(repo_helper.repo, "master", "branch2", "file3.txt", "content3",
"Branch2 commit") # noqa: F841
repo_helper.enable_caching()
repo_helper.set_desired_branches([("branch1", "HEAD"), ("branch2", "HEAD")])
repo_helper.progress()
repo_helper.verify_working_directory_files([
('file1.txt', 'content1'),
('file2.txt', 'content2'),
('file3.txt', 'content3'),
])
assert repo_helper.checkout_merged.progress().version == '-'.join([it[:8] for it in sorted([commit2.hexsha, commit3.hexsha])])
repo_helper.remove_branch(repo_helper.repo, "branch2")
# CheckoutMerged should raise an error when it can't find the branch, and ignore_missing_branches is False
get_step(repo_helper.env.pipeline, CheckoutMerged).ignore_missing_branches = False
with pytest.raises(BadName, match="Ref 'origin/branch2' did not resolve to an object"):
repo_helper.progress()
assert repo_helper.get_desired_branches() == [("branch1", "HEAD"), ("branch2", "HEAD")]
with pytest.raises(BadName, match="Ref 'origin/branch2' did not resolve to an object"):
repo_helper.checkout_merged.progress()
# Now set ignore_missing_branches to True and verify that it ignores the missing branch and continues processing
get_step(repo_helper.env.pipeline, CheckoutMerged).ignore_missing_branches = True
repo_helper.progress()
assert repo_helper.get_desired_branches() == [("branch1", "HEAD"), ("branch2", "HEAD")]
assert repo_helper.checkout_merged.progress().version == '-'.join([it[:8] for it in [commit2.hexsha]])
# Removing missing branch should be processed well
repo_helper.set_desired_branches([("branch1", "HEAD")])
repo_helper.progress()
assert repo_helper.checkout_merged.progress().version == '-'.join([it[:8] for it in [commit2.hexsha]])
repo_helper.verify_working_directory_files([
('file1.txt', 'content1'),
('file2.txt', 'content2'),
])
+7 -27
View File
@@ -10,8 +10,8 @@ import tempfile
from typing import Dict, Tuple, List, Callable, cast
import git
from enironment import Environment, AbstractStep, wrap_in_cached, get_step, SharedStateHolder
from processing import reset_caches
from enironment import Environment, AbstractStep, SharedStateHolder, get_step
from steps.docker import DockerSwarmCheckResult
from steps.git import CheckoutAndMergeResult, CheckoutMerged, GitUnmerge, GitClone, HasVersion, GitUnmergeResult
from steps.shared_state import SharedStateHolderInMemory
@@ -106,22 +106,6 @@ class RemoteRepoHelper:
def mock_check(self) -> AbstractStep[Dict[str, HasVersion]]:
return get_step(self.env.pipeline, MockDockerSwarmCheck)
def enable_caching(self) -> None:
"""Wrap environment pipeline in caching steps"""
self.env = wrap_in_cached(self.env)
def progress(self) -> None:
reset_caches([self.env])
for i in self.env.pipeline:
i.progress()
def remove_branch(self, repo: git.Repo, branch_name: str) -> None:
"""Remove a branch from the repository"""
if branch_name in [head.name for head in repo.heads]:
repo.delete_head(branch_name, force=True)
else:
raise BaseException(f"Branch doesn't exist {branch_name}.")
def teardown(self) -> None:
"""Clean up temporary directories"""
if self.remote_dir:
@@ -158,9 +142,7 @@ class RemoteRepoHelper:
with open(file_path, 'w') as f:
f.write(content)
repo.index.add([filename])
result = repo.index.commit(message)
repo.git.checkout(result.hexsha, detach=True)
return result
return repo.index.commit(message)
def verify_working_directory_files(self, expected_files: List[Tuple[str, str]]) -> None:
"""Verify that working directory contains exactly the expected files with correct content.
@@ -186,12 +168,10 @@ class RemoteRepoHelper:
repo = git.Repo(repo_path)
commits = list(repo.iter_commits("--all", max_count=30))
refs = []
for it in repo.references:
try:
refs.append(f"{_as_text(it.commit)[:10]}:{_as_text(it.name)}, is_remote:{it.is_remote()}")
except Exception as e:
refs.append(f"Error reading branch {it.name}: {e}")
refs = [
f"{_as_text(it.commit)[:10]}:{_as_text(it.name)}, is_remote:{it.is_remote()}"
for it in repo.refs
]
lines = [
f"{_as_text(commit.hexsha)[:10]} (p{[_as_text(it.hexsha)[:10] for it in commit.parents]}): {_as_text(commit.message).splitlines()[0]}"
for commit in commits