diff --git a/backend/configs/backups_helper.py b/backend/configs/backups_helper.py new file mode 100644 index 0000000..d648ad4 --- /dev/null +++ b/backend/configs/backups_helper.py @@ -0,0 +1,61 @@ +from steps.git import GitClone, CheckoutMerged, GitUnmerge +from steps.docker import DockerSwarmCheck, DockerSwarmDeploy +from enironment import Environment +from steps.checks import SimpleLog, UrlCheck +from steps.shared_state import SharedStateHolderInMemory + + + +clone = GitClone(url="https://github.com/rudolf1/uber_backup.git", branchNamePrefix="photosHelper") + +sharedState = SharedStateHolderInMemory(unmerge=None) + +checkoutMerged = CheckoutMerged(clone, + desired_branches=sharedState, + push = False, + git_user_email="rudolfss13@gmail.com", + git_user_name="brencher_bot" + ) + +sharedState.set_branches([("photosHelper/master", "HEAD")]) +dockerSwarmCheck = DockerSwarmCheck( + stack_name = "photosHelper", +) +deployDocker = DockerSwarmDeploy( + wd=clone, + buildDocker=None, + stackChecker=dockerSwarmCheck, + envs = lambda: { + "version": "auto-" + checkoutMerged.progress().version + }, + stack_name = "photosHelper", + docker_compose_path = "docker-compose.yml", +) +unmerge = GitUnmerge(clone, dockerSwarmCheck) + +checkPing = UrlCheck( + url="https://backuper.rudolf.keenetic.link/api/server/ping", + expected = {"res":"pong"}, +) +logUrls = SimpleLog(message = { + "userLinks": { + "App": "https://backuper.rudolf.keenetic.link", + } +}) + +backups_helper = Environment( + id="backupsHelper", + state=sharedState, + pipeline=[ + clone, + sharedState, + checkoutMerged, + dockerSwarmCheck, + unmerge, + deployDocker, + checkPing, + logUrls + ] +) + + diff --git a/backend/steps/docker.py b/backend/steps/docker.py index 3cdedaa..d087f39 100644 --- a/backend/steps/docker.py +++ b/backend/steps/docker.py @@ -99,7 +99,7 @@ class DockerComposeBuild(AbstractStep[Dict[str, str]]): pass logger.info(f"Building image {image} from {build_ctx}, {build_dockerfile}") - img, _ = client.images.build(path=build_ctx, dockerfile=build_dockerfile, tag=image, nocache=not self.build_cache, rm=True) + img, _ = client.images.build(path=build_ctx, dockerfile=build_dockerfile, tag=image, nocache=not self.build_cache, rm=not self.build_cache) if self.publish: logger.info(f"Pushing image {image}") diff --git a/backend/steps/git.py b/backend/steps/git.py index d06ce2e..9fe526d 100644 --- a/backend/steps/git.py +++ b/backend/steps/git.py @@ -119,6 +119,7 @@ class CheckoutMerged(AbstractStep[CheckoutAndMergeResult]): desired_branches: AbstractStep[SharedState], git_user_email: str, git_user_name: str, + ignore_missing_branches: bool = True, push: bool = True, **kwargs: Any): super().__init__(**kwargs) @@ -127,7 +128,7 @@ class CheckoutMerged(AbstractStep[CheckoutAndMergeResult]): self.git_user_email = git_user_email self.git_user_name = git_user_name self.push = push - + self.ignore_missing_branches = ignore_missing_branches def _find_merge_childs(self, tree: Dict[Commit, List[Commit]], commit: Commit) -> List[Commit]: visited = [commit] @@ -155,6 +156,8 @@ class CheckoutMerged(AbstractStep[CheckoutAndMergeResult]): commit = repo.commit(desired_commit) commit_ids[commit] = branch_name except BaseException as e: + if not self.ignore_missing_branches: + raise e stack = traceback.format_exception(type(e), e, e.__traceback__) logger.error(f"Error finding desired commits for branch {branch_pair}: {str(e)}\n{''.join(stack)}") return commit_ids @@ -229,7 +232,7 @@ class CheckoutMerged(AbstractStep[CheckoutAndMergeResult]): version = '-'.join([x.hexsha[0:8] for x in sorted_commits]) for ref in repo.refs: - if ref.is_remote() and ref.commit == repo.head.commit and ref.name != 'origin/HEAD': + if ref.is_remote() and ref.is_valid() and ref.commit == repo.head.commit and ref.name != 'origin/HEAD': logger.info(f"Merge commit {ref.commit} corresponds to branch {ref}") remote_branch_name = ref.name[len('origin/'):] break @@ -293,9 +296,9 @@ class GitUnmerge(AbstractStep[GitUnmergeResult]): for v in version_parts: commit = repo.commit(v) # branches = [head.name for head in repo.heads if head.commit.hexsha == commit.hexsha] - branches = [ref.name for ref in repo.remotes.origin.refs if ref.commit.hexsha == commit.hexsha] + branches = [ref.name for ref in repo.remotes.origin.refs if not ref.name.startswith('origin/HEAD') and ref.commit.hexsha == commit.hexsha] branches = [b[len('origin/'):] for b in branches if - b.startswith('origin/') and not b.startswith('origin/HEAD')] + b.startswith('origin/') ] if len(branches) == 0 or branches[0].startswith('auto/'): if childs is None: diff --git a/backend/steps/step.py b/backend/steps/step.py index bd364fc..f8d3c99 100644 --- a/backend/steps/step.py +++ b/backend/steps/step.py @@ -60,3 +60,10 @@ class CachingStep(AbstractStep[T], Generic[T]): def __getattr__(self, name: str) -> Any: """Delegate unknown attributes to the wrapped step.""" return getattr(self._step, name) + + def __setattr__(self, name: str, value: Any) -> None: + """Delegate unknown attributes to the wrapped step.""" + if name in {"name", "_step", "_result", "_input_hash"}: + super().__setattr__(name, value) + else: + setattr(self._step, name, value) diff --git a/docker-compose.yml b/docker-compose.yml index 5f1ed92..6a2cdb0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,7 +16,7 @@ services: - PYTHONUNBUFFERED=1 - VIRTUAL_HOST=brencher.rudolf.keenetic.link - LETSENCRYPT_HOST=brencher.rudolf.keenetic.link - - PROFILES=brencher2,torrserv_proxy,immich,registry,gmail_mcp + - PROFILES=brencher2,torrserv_proxy,immich,registry,gmail_mcp,backupsHelper - SECONDARY_BRENCHER=http://host.docker.internal:5002 deploy: replicas: 1 diff --git a/tests/test_git_remove_branch.py b/tests/test_git_remove_branch.py new file mode 100644 index 0000000..d9e4cd6 --- /dev/null +++ b/tests/test_git_remove_branch.py @@ -0,0 +1,61 @@ +from git import BadName +import pytest + +from enironment import get_step +from steps.git import CheckoutMerged +from tests.conftest import repo_helper + +from .test_remote_repo import RemoteRepoHelper + + +class TestGitRemoveBranch: + """Integration tests for git operations""" + + def test_checkout_merged_one_branch(self, repo_helper: RemoteRepoHelper) -> None: + # Create main branch with initial commit + commit1 = repo_helper.create_commit(repo_helper.repo, "master", "master", "file1.txt", "content1", + "Initial commit") # noqa: F841 + commit2 = repo_helper.create_commit(repo_helper.repo, "master", "branch1", "file2.txt", "content2", + "Branch1 commit") # noqa: F841 + commit3 = repo_helper.create_commit(repo_helper.repo, "master", "branch2", "file3.txt", "content3", + "Branch2 commit") # noqa: F841 + + repo_helper.enable_caching() + + repo_helper.set_desired_branches([("branch1", "HEAD"), ("branch2", "HEAD")]) + repo_helper.progress() + + repo_helper.verify_working_directory_files([ + ('file1.txt', 'content1'), + ('file2.txt', 'content2'), + ('file3.txt', 'content3'), + ]) + + assert repo_helper.checkout_merged.progress().version == '-'.join([it[:8] for it in sorted([commit2.hexsha, commit3.hexsha])]) + + repo_helper.remove_branch(repo_helper.repo, "branch2") + + # CheckoutMerged should raise an error when it can't find the branch, and ignore_missing_branches is False + get_step(repo_helper.env.pipeline, CheckoutMerged).ignore_missing_branches = False + + with pytest.raises(BadName, match="Ref 'origin/branch2' did not resolve to an object"): + repo_helper.progress() + assert repo_helper.get_desired_branches() == [("branch1", "HEAD"), ("branch2", "HEAD")] + with pytest.raises(BadName, match="Ref 'origin/branch2' did not resolve to an object"): + repo_helper.checkout_merged.progress() + + # Now set ignore_missing_branches to True and verify that it ignores the missing branch and continues processing + get_step(repo_helper.env.pipeline, CheckoutMerged).ignore_missing_branches = True + repo_helper.progress() + assert repo_helper.get_desired_branches() == [("branch1", "HEAD"), ("branch2", "HEAD")] + assert repo_helper.checkout_merged.progress().version == '-'.join([it[:8] for it in [commit2.hexsha]]) + + # Removing missing branch should be processed well + repo_helper.set_desired_branches([("branch1", "HEAD")]) + repo_helper.progress() + assert repo_helper.checkout_merged.progress().version == '-'.join([it[:8] for it in [commit2.hexsha]]) + + repo_helper.verify_working_directory_files([ + ('file1.txt', 'content1'), + ('file2.txt', 'content2'), + ]) diff --git a/tests/test_remote_repo.py b/tests/test_remote_repo.py index 009856a..5176d36 100644 --- a/tests/test_remote_repo.py +++ b/tests/test_remote_repo.py @@ -10,8 +10,8 @@ import tempfile from typing import Dict, Tuple, List, Callable, cast import git - -from enironment import Environment, AbstractStep, SharedStateHolder, get_step +from enironment import Environment, AbstractStep, wrap_in_cached, get_step, SharedStateHolder +from processing import reset_caches from steps.docker import DockerSwarmCheckResult from steps.git import CheckoutAndMergeResult, CheckoutMerged, GitUnmerge, GitClone, HasVersion, GitUnmergeResult from steps.shared_state import SharedStateHolderInMemory @@ -106,6 +106,22 @@ class RemoteRepoHelper: def mock_check(self) -> AbstractStep[Dict[str, HasVersion]]: return get_step(self.env.pipeline, MockDockerSwarmCheck) + def enable_caching(self) -> None: + """Wrap environment pipeline in caching steps""" + self.env = wrap_in_cached(self.env) + + def progress(self) -> None: + reset_caches([self.env]) + for i in self.env.pipeline: + i.progress() + + def remove_branch(self, repo: git.Repo, branch_name: str) -> None: + """Remove a branch from the repository""" + if branch_name in [head.name for head in repo.heads]: + repo.delete_head(branch_name, force=True) + else: + raise BaseException(f"Branch doesn't exist {branch_name}.") + def teardown(self) -> None: """Clean up temporary directories""" if self.remote_dir: @@ -142,7 +158,9 @@ class RemoteRepoHelper: with open(file_path, 'w') as f: f.write(content) repo.index.add([filename]) - return repo.index.commit(message) + result = repo.index.commit(message) + repo.git.checkout(result.hexsha, detach=True) + return result def verify_working_directory_files(self, expected_files: List[Tuple[str, str]]) -> None: """Verify that working directory contains exactly the expected files with correct content. @@ -168,10 +186,12 @@ class RemoteRepoHelper: repo = git.Repo(repo_path) commits = list(repo.iter_commits("--all", max_count=30)) - refs = [ - f"{_as_text(it.commit)[:10]}:{_as_text(it.name)}, is_remote:{it.is_remote()}" - for it in repo.refs - ] + refs = [] + for it in repo.references: + try: + refs.append(f"{_as_text(it.commit)[:10]}:{_as_text(it.name)}, is_remote:{it.is_remote()}") + except Exception as e: + refs.append(f"Error reading branch {it.name}: {e}") lines = [ f"{_as_text(commit.hexsha)[:10]} (p{[_as_text(it.hexsha)[:10] for it in commit.parents]}): {_as_text(commit.message).splitlines()[0]}" for commit in commits