diff --git a/.github/workflows/merge_main.yaml b/.github/workflows/merge_main.yaml index 4ac593b..770e8d3 100644 --- a/.github/workflows/merge_main.yaml +++ b/.github/workflows/merge_main.yaml @@ -20,16 +20,16 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 - token: ${{ secrets.GITHUB_TOKEN }} + token: ${{ secrets.PUSH_TOKEN }} - name: Configure git run: | - git config user.email "github-actions[bot]@users.noreply.github.com" - git config user.name "GitHub Actions" + git config user.email "gitea-actions[bot]@noreply.localhost" + git config user.name "Gitea Actions" - name: Fetch all branches run: | - git remote set-url origin "https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}.git" + git remote set-url origin "https://x-access-token:${PUSH_TOKEN}@gitea.rudolf.keenetic.link/${GITHUB_REPOSITORY}.git" git fetch --all --prune - name: Merge every branch onto main @@ -37,7 +37,6 @@ jobs: run: | set -eo pipefail MAIN=main - # Collect remote branch names (exclude HEAD and main) branches=$(git for-each-ref --format='%(refname:short)' refs/remotes/origin/ \ | sed 's|^origin/||' \ | grep -v -e '^HEAD$' -e '^main$' -e '^origin$' \ @@ -49,11 +48,8 @@ jobs: continue fi echo "=== Processing branch: $br ===" - # Create/update a local branch from remote git checkout -B "$br" "origin/$br" - # Merge origin/main into branch (non-interactive) if git merge --no-edit "origin/$MAIN"; then - # Push updated branch to remote (no force) if git push origin "$br"; then echo "Pushed merged $br" else diff --git a/.github/workflows/merge_main_and_squash.yaml b/.github/workflows/merge_main_and_squash.yaml index 968adbc..2034229 100644 --- a/.github/workflows/merge_main_and_squash.yaml +++ b/.github/workflows/merge_main_and_squash.yaml @@ -18,16 +18,16 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 - token: ${{ secrets.GITHUB_TOKEN }} + token: ${{ secrets.PUSH_TOKEN }} - name: Configure git run: | - git config user.email "github-actions[bot]@users.noreply.github.com" - git config user.name "GitHub Actions" + git config user.email "gitea-actions[bot]@noreply.localhost" + git config user.name "Gitea Actions" - name: Fetch all branches run: | - git remote set-url origin "https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}.git" + git remote set-url origin "https://x-access-token:${PUSH_TOKEN}@gitea.rudolf.keenetic.link/${GITHUB_REPOSITORY}.git" git fetch --all --prune - name: Merge every branch onto main @@ -35,7 +35,6 @@ jobs: run: | set -eo pipefail MAIN=main - # Collect remote branch names (exclude HEAD and main) branches=$(git for-each-ref --format='%(refname:short)' refs/remotes/origin/ \ | sed 's|^origin/||' \ | grep -v -e '^HEAD$' -e '^main$' -e '^origin$' \ @@ -47,12 +46,9 @@ jobs: continue fi echo "=== Processing branch: $br ===" - # Create/update a local branch from remote git reset --hard HEAD git checkout -B "$br" "origin/$br" - # Merge origin/main into branch (non-interactive) if git merge --no-edit --squash "origin/$MAIN"; then - # Push updated branch to remote (no force) git reset --soft "origin/$MAIN" git commit -m "Squash" if git push origin "$br" --force-with-lease; then diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index dd1317b..6d55d96 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -1,6 +1,3 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions -# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python - name: Python package on: @@ -10,74 +7,40 @@ on: pull_request: branches: [ "main" ] +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: build: runs-on: ubuntu-latest strategy: fail-fast: false matrix: - python-version: [ - # "3.11.3", - "3.12.3", - "3.13", - "3.14", - # "3.15", - ] - checker: [ - "python -c \"import sys; print(f'Python {sys.version}'); print(f'Build: {sys.version_info}'); print(f'Implementation: {sys.implementation}')\"", - "pip list", - "flake8 . --count --select=E9,F63,F7,F82 --ignore=F824 --show-source --statistics", - "flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics", - # "ty check backend", - "mypy --no-incremental --config-file mypy.ini backend tests", - ] + python-version: ["3.12.3", "3.13", "3.14"] steps: - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }}, ${{ matrix.checker }} - uses: actions/setup-python@v3 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + cache: 'pip' + cache-dependency-path: | + requirements.txt + requirements-dev.txt + - name: Install dependencies run: | python -m pip install --upgrade pip pip install -r requirements.txt pip install -r requirements-dev.txt - - name: Run checker - run: ${{ matrix.checker }} -# tests: -# runs-on: ubuntu-latest -# needs: build -# strategy: -# fail-fast: false -# matrix: -# python-version: [ -# # "3.11.3", -# "3.12.3", -# "3.13", -# "3.14", -# # "3.15", -# ] -# checker: [ -# "pytest", -# ] -# steps: -# - uses: actions/checkout@v4 -# - name: Set up Python ${{ matrix.python-version }}, ${{ matrix.checker }} -# uses: actions/setup-python@v3 -# with: -# python-version: ${{ matrix.python-version }} -# - name: Install dependencies -# run: | -# # sudo apt-get update -# # sudo apt-get upgrade -y -# # sudo apt-get install -y --no-install-recommends git docker.io -# python -m pip install --upgrade pip -# pip install -r requirements-dev.txt -# playwright install --with-deps chromium -# - name: Run checker -# run: "pytest" -# -# -# + - name: Lint (flake8) + run: | + flake8 . --count --select=E9,F63,F7,F82 --ignore=F824 --show-source --statistics + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + + - name: Type-check (mypy) + run: mypy --no-incremental --config-file mypy.ini backend tests \ No newline at end of file diff --git a/REST.instruction.md b/REST.instruction.md index 8001cb0..5869158 100644 --- a/REST.instruction.md +++ b/REST.instruction.md @@ -17,3 +17,6 @@ state: str # 'Active' or 'Pause' pipeline: Dict[str, Any] + + + diff --git a/backend/app.py b/backend/app.py index 0ced9f9..63cc7ba 100644 --- a/backend/app.py +++ b/backend/app.py @@ -48,7 +48,7 @@ class App: "name": r.name, "status": [str(result), stack], "error": True, - "is_running": True, + "is_running": False, }) else: pipeline_state.append({ @@ -62,7 +62,7 @@ class App: "name": r.name, "status": [str(e), stack], "error": True, - "is_running": True, + "is_running": False, }) env_dtos[env.id] = {'id': env.id } env_dtos[env.id]['pipeline'] = pipeline_state diff --git a/backend/configs/hermes_infra.py b/backend/configs/hermes_infra.py index 852ba9f..a5ab2e3 100644 --- a/backend/configs/hermes_infra.py +++ b/backend/configs/hermes_infra.py @@ -4,7 +4,28 @@ from steps.docker import DockerComposeBuild, DockerSwarmCheck, DockerSwarmDeploy from steps.git import GitClone, CheckoutMerged, GitUnmerge from steps.shared_state import SharedStateHolderInMemory -clone = GitClone(url="https://github.com/rudolf1/uber_backup.git", branchNamePrefix="ansible") +# Dual-remote clone for the ansible mirror (the deploy target, not brencher itself). +# - gitea_hermes is where the feature branch lives (hermes/ansible:*) +# - gitea_admin is the live-deploy source (where brencher-bot reads compose files from) +# +# Why not github? github.com is unreachable from the brencher container -- the +# proxy returns HTTP 403 on CONNECT tunnel, so the URL would always fail to +# fetch. See backend/configs/docker_socket_proxy.py for the same migration. +# +# Why drop branchNamePrefix? GitClone only applies its prefix refspec to a +# remote named "origin" (backend/steps/git.py ~L104); neither remote here is +# named "origin", so the prefix would be a silent no-op. Both remotes fetch +# their full refset instead. +# +# Why two remotes? The deploy step (DockerSwarmDeploy) checks out a specific +# branch and uses the resulting checkout as wd for the compose path. With +# gitea_hermes as primary, feature branches opened in the fork are picked up; +# gitea_admin provides the live-deploy source as a fallback / for cross-fork +# PRs. This matches the form in backend/configs/brencher.py after PR #3. +clone = GitClone(remotes=[ + {"name": "gitea_hermes", "url": "https://gitea.rudolf.keenetic.link/hermes/ansible"}, + {"name": "gitea_admin", "url": "https://gitea.rudolf.keenetic.link/admin/ansible"}, +]) dockerSwarmCheck = DockerSwarmCheck( stack_name="hermes_infra", diff --git a/backend/configs/torrserv_proxy.py b/backend/configs/torrserv_proxy.py index 16db1ef..7fe964d 100644 --- a/backend/configs/torrserv_proxy.py +++ b/backend/configs/torrserv_proxy.py @@ -3,7 +3,11 @@ from steps.docker import DockerComposeBuild, DockerSwarmCheck, DockerSwarmDeploy from steps.git import GitClone, CheckoutMerged, GitUnmerge from steps.shared_state import SharedStateHolderInMemory -clone = GitClone(url="https://github.com/rudolf1/reverseproxy.git") +clone = GitClone(remotes=[ + {"name": "github", "url": "https://github.com/rudolf1/reverseproxy.git"}, + {"name": "gitea_admin", "url": "https://gitea.rudolf.keenetic.link/admin/reverseproxy"}, + {"name": "gitea_hermes", "url": "https://gitea.rudolf.keenetic.link/hermes/reverseproxy"}, +]) dockerSwarmCheck = DockerSwarmCheck( stack_name="torrserv_proxy", diff --git a/backend/steps/docker.py b/backend/steps/docker.py index e9587c7..9c08d81 100644 --- a/backend/steps/docker.py +++ b/backend/steps/docker.py @@ -1,3 +1,4 @@ +import hashlib import logging import os import re @@ -197,6 +198,32 @@ class DockerSwarmDeploy(AbstractStep[str]): content = f.read() content = re.sub(r'\$\{([^}]+)\}', lambda m: env.get(m.group(1), ""), content) compose = yaml.safe_load(content) + version = env["version"] + if "configs" in compose: + sanitized_version = re.sub(r'[^a-zA-Z0-9_.-]', '_', version) + full_prefix = f"{self.stack_name}_" + + def _config_full_name(base_name: str) -> str: + full = f"{full_prefix}{base_name}_{sanitized_version}" + if len(full) <= 64: + return f"{base_name}_{sanitized_version}" + return hashlib.sha256(full.encode("utf-8")).hexdigest()[:32] + + config_names = { + name: _config_full_name(name) + for name in compose["configs"] + } + compose["configs"] = { + config_names.get(name, name): config + for name, config in compose["configs"].items() + } + for service in compose.get("services", {}).values(): + for config in service.get("configs", []): + if isinstance(config, str): + config_index = service["configs"].index(config) + service["configs"][config_index] = config_names.get(config, config) + elif isinstance(config, dict) and "source" in config: + config["source"] = config_names.get(config["source"], config["source"]) if "services" in compose: for svc in compose["services"].values(): if "build" in svc: @@ -268,6 +295,7 @@ class DockerSwarmDeploy(AbstractStep[str]): # Deploy stack to Docker Swarm cmd = [ "docker", "stack", "deploy", + "--detach=false", "--prune", "-c", tmp_compose_path, self.stack_name @@ -282,9 +310,24 @@ class DockerSwarmDeploy(AbstractStep[str]): result = subprocess.run(cmd, capture_output=True, text=True, cwd=os.path.dirname(tmp_compose_path)) #, env=swarmEnv) os.remove(tmp_compose_path) - if result.returncode != 0: - logger.error(f"Stack deploy failed: {result.stderr}") - raise RuntimeError(f"Stack deploy failed: {result.stderr}") + logger.info(f"docker stack deploy stdout: {result.stdout or ''}") + logger.info(f"docker stack deploy stderr: {result.stderr or ''}") + stderr_lower = result.stderr.lower() + docker_error_markers = ( + "failed to ", + "error response from daemon", + "rpc error", + "denied", + "no such image", + ) + if result.returncode != 0 or any(marker in stderr_lower for marker in docker_error_markers): + deploy_output = ( + f"exit code: {result.returncode}\n" + f"stdout:\n{result.stdout or ''}\n" + f"stderr:\n{result.stderr or ''}" + ) + logger.error(f"Stack deploy failed:\n{deploy_output}") + raise RuntimeError(f"Stack deploy failed:\n{deploy_output}") logger.info(f"Stack deployed successfully: {result.stdout}") # Check health of all deployed services @@ -308,4 +351,25 @@ class DockerSwarmDeploy(AbstractStep[str]): if not_running: raise BaseException(f"Services not started yet: {', '.join(not_running)}") + version_mismatches = [] + for svc_name, svc in expected_services.items(): + full_svc_name = f"{self.stack_name}_{svc_name}" + services_list = client.services.list(filters={"name": full_svc_name}) + if not services_list: + continue + service_labels = services_list[0].attrs.get("Spec", {}).get("TaskTemplate", {}).get("ContainerSpec", {}).get("Labels", {}) + expected_version = svc.get("labels", {}).get("org.brencher.version") + actual_version = service_labels.get("org.brencher.version") + if expected_version != actual_version: + version_mismatches.append( + f"{full_svc_name} expected version {expected_version}, got {actual_version}" + ) + update_status = services_list[0].attrs.get("UpdateStatus", {}) + if update_status.get("State") in {"paused", "rollback_paused", "rollback_started"}: + version_mismatches.append( + f"{full_svc_name} Swarm update {update_status.get('State')}: {update_status.get('Message', '')}" + ) + if version_mismatches: + raise RuntimeError(f"Deployment did not apply requested versions: {'; '.join(version_mismatches)}") + return f"Stack deployed successfully: {result.stdout}" diff --git a/requirements-dev.txt b/requirements-dev.txt index 734d36c..8a92c00 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -11,3 +11,7 @@ types-PyYAML types-requests types-docker testcontainers + + + +