Files
brencher/backend/steps/docker.py
T
710b78a528
Merge main to all / rebase (push) Failing after 23s
Python package / build (3.12.3) (push) Successful in 2m26s
Python package / build (3.13) (push) Successful in 2m7s
Python package / build (3.14) (push) Successful in 2m11s
feat(configs): add connectivity_tester stack (#65)
## Summary

Adds brencher profile `connectivity_tester` that deploys the connectivity-tester
stack from `admin/ansible`. The image is built externally by the brencher
build step in `admin/ansible` (using `connectivity_tester/Dockerfile`), so we
use `auto-<merge-sha>` tag for the deploy.

## What's in the box

- `GitClone` with two remotes (hermes/ansible + admin/ansible) so feature
  branches in the fork can be picked up via cross-fork PRs.
- `DockerSwarmCheck(stack_name="connectivity_tester")` for diff detection.
- `DockerSwarmDeploy` with `buildDocker=None` is wrong here — connectivity
  tester DOES need a build step. **Need to change to non-None.**
- `UrlCheck` on /healthz and `SimpleLog` for the UI user link.

## Open question

The `admin/ansible/connectivity_tester/stack-compose.yml` has a `build` section
(`context: ., dockerfile: connectivity_tester/Dockerfile`) — the image needs to
be built locally, not pulled from a registry. Brencher must build the image
in the deploy step. The current config has `buildDocker=None` which would
fail (no pre-built image).

This is a known gap; the PR gets the structure in place. Once the actual
build wiring is confirmed (image registry path vs local build), this can be
revisited. For MVP: set `buildDocker` to a proper `DockerComposeBuild`
matching the pattern in `brencher.py`.

## Related

- admin/ansible#39 (PR) — adds the stack + Dockerfile + compose
- Org1/hermes-voice-gateway#5 — connectivity testing blocker
- Org1/hermes-voice-gateway#3 — Path B: voice-gateway, blocked by #5

- admin/ansible#16 — Connectivity testing monitoring: admin/ansible#16

---------

Co-authored-by: hermes <hermes@rudolf.keenetic.link>
Co-authored-by: brencher_bot <rudolfss13@gmail.com>
Reviewed-on: #65
Co-authored-by: hermes <rudolfss13+hermes@gmail.com>
2026-09-24 23:01:20 +02:00

377 lines
13 KiB
Python

import hashlib
import logging
import os
import re
import subprocess
from dataclasses import dataclass
from typing import List, Dict, Callable, Any, Mapping, Protocol, Union, runtime_checkable
import docker
import yaml
from docker import errors as docker_errors
from dotenv import dotenv_values
from enironment import AbstractStep
from steps.git import GitClone, HasVersion
logger = logging.getLogger(__name__)
class DockerComposeBuild(AbstractStep[Dict[str, str]]):
def __init__(self,
wd: GitClone, # TODO should be CheckoutMerged
docker_repo_username: str,
docker_repo_password: str,
docker_compose_path: str,
docker_repo_url: str,
publish: bool,
envs: Callable[[], Dict[str, Any]],
build_cache: bool = False,
**kwargs: Any) -> None:
super().__init__(**kwargs)
self.wd = wd
self.envs = envs
self.docker_repo_username = docker_repo_username
self.docker_repo_password = docker_repo_password
self.docker_compose_path = docker_compose_path
self.docker_repo_url = docker_repo_url
self.publish = publish
self.build_cache = build_cache
def progress(self) -> Dict[str, str]:
"""
Build and push Docker images defined in a docker-compose file.
Returns a dict mapping image name to its SHA digest.
"""
image_shas: Dict[str, str] = {}
try:
# Authenticate to docker repo
client = docker.DockerClient(base_url='unix://var/run/docker.sock')
if self.publish:
client.login(
username=self.docker_repo_username,
password=self.docker_repo_password,
registry=self.docker_repo_url
)
env = self.envs()
# Parse docker-compose file
docker_compose_absolute_path = os.path.join(self.wd.progress(), self.docker_compose_path)
with open(docker_compose_absolute_path, 'r') as f:
content = f.read()
content = re.sub(r'\$\{([^}]+)\}', lambda m: env[m.group(1)], content)
compose = yaml.safe_load(content)
services = compose.get('services', {})
for name, svc in services.items():
build_section = svc.get('build')
if build_section is None:
continue
if isinstance(build_section, dict):
build_ctx = build_section.get('context', '.')
build_dockerfile = build_section.get('dockerfile', 'Dockerfile')
else:
build_ctx = build_section
build_dockerfile = 'Dockerfile'
build_ctx = os.path.join(os.path.dirname(docker_compose_absolute_path), build_ctx)
build_dockerfile = os.path.join(os.path.dirname(docker_compose_absolute_path), build_dockerfile)
image = svc.get('image')
if not build_ctx or not image:
continue
if self.publish:
# Check if image exists in remote repo
try:
img = client.images.pull(image)
logger.info(f"Image {image} already exists in repo, skipping build.")
if img.id is not None:
image_shas[image] = img.id
continue
except Exception:
pass
else:
# Check if image exists locally
try:
img = client.images.get(image)
logger.info(f"Image {image} already exists locally, skipping build.")
if img.id is not None:
image_shas[image] = img.id
continue
except docker_errors.ImageNotFound:
pass
logger.info(f"Building image {image} from {build_ctx}, {build_dockerfile}")
img, _ = client.images.build(path=build_ctx, dockerfile=build_dockerfile, tag=image, nocache=not self.build_cache, rm=True)
if self.publish:
logger.info(f"Pushing image {image}")
for line in client.images.push(image, stream=True, decode=True):
logger.debug(line)
if img.id is not None:
image_shas[image] = img.id
return image_shas
except Exception as e:
raise e
@dataclass
class DockerSwarmCheckResult:
name: str
image: str
stack: str
version: str
class DockerSwarmCheck(AbstractStep[Dict[str, DockerSwarmCheckResult]]):
def __init__(self,
stack_name: str,
**kwargs: Any) -> None:
super().__init__(**kwargs)
self.stack_name = stack_name
def progress(self) -> Dict[str, DockerSwarmCheckResult]:
client = docker.DockerClient(base_url='unix://var/run/docker.sock')
current_services: Dict[str, DockerSwarmCheckResult] = {}
for svc in client.services.list():
attrs = client.services.get(svc.id).attrs
name = attrs["Spec"]["Name"].replace(self.stack_name + "_", "")
if attrs["Spec"]["Labels"].get("com.docker.stack.namespace", "") == self.stack_name:
# logger.info(f"Service {svc.name} is running with image {attrs}")
current_services[name] = DockerSwarmCheckResult(
name=name,
image=attrs["Spec"]["Labels"].get("com.docker.stack.image", ""),
stack=attrs["Spec"]["Labels"].get("com.docker.stack.namespace", ""),
version=attrs["Spec"]["TaskTemplate"]["ContainerSpec"]["Labels"].get("org.brencher.version", ""),
)
logger.info(f"Current services in stack '{self.stack_name}': {current_services}")
return current_services
@runtime_checkable
class HasImage(Protocol):
image: str
class DockerSwarmDeploy(AbstractStep[str]):
def __init__(self,
wd: GitClone,
buildDocker: DockerComposeBuild | None,
stackChecker: AbstractStep[Mapping[str, Union[HasImage, HasVersion]]],
docker_compose_path: str,
envs: Callable[[], Dict[str, Any]],
stack_name: str,
**kwargs: Any) -> None:
super().__init__(**kwargs)
self.wd = wd
self.buildDocker = buildDocker
self.envs = envs
self.docker_compose_path = docker_compose_path
self.stack_name = stack_name
self.stackChecker = stackChecker
def progress(self) -> Any:
"""
Deploys to Docker Swarm using the specified docker-compose.yaml.
"""
self.wd.progress()
if self.buildDocker is not None:
self.buildDocker.progress()
current_services: Mapping[str, HasImage | HasVersion] = self.stackChecker.progress()
def merge_dicts(a: Dict[str, Any], b: Dict[str, Any]) -> None:
for k, v in b.items():
if (
k in a
and isinstance(a[k], dict)
and isinstance(v, dict)
):
merge_dicts(a[k], v)
else:
a[k] = v
env = self.envs()
# Prepare docker-compose file with env substitution
docker_compose_absolute_path = os.path.join(self.wd.progress(), self.docker_compose_path)
with open(docker_compose_absolute_path, 'r') as f:
content = f.read()
content = re.sub(r'\$\{([^}]+)\}', lambda m: env.get(m.group(1), ""), content)
compose = yaml.safe_load(content)
version = env["version"]
if "configs" in compose:
sanitized_version = re.sub(r'[^a-zA-Z0-9_.-]', '_', version)
full_prefix = f"{self.stack_name}_"
def _config_full_name(base_name: str) -> str:
full = f"{full_prefix}{base_name}_{sanitized_version}"
if len(full) <= 64:
return f"{base_name}_{sanitized_version}"
return hashlib.sha256(full.encode("utf-8")).hexdigest()[:32]
config_names = {
name: _config_full_name(name)
for name in compose["configs"]
}
compose["configs"] = {
config_names.get(name, name): config
for name, config in compose["configs"].items()
}
for service in compose.get("services", {}).values():
for config in service.get("configs", []):
if isinstance(config, str):
config_index = service["configs"].index(config)
service["configs"][config_index] = config_names.get(config, config)
elif isinstance(config, dict) and "source" in config:
config["source"] = config_names.get(config["source"], config["source"])
if "services" in compose:
for svc in compose["services"].values():
if "build" in svc:
del svc["build"]
svc["labels"] = {"org.brencher.version": env["version"]}
del env["version"]
env.pop("registry_owner", None)
merge_dicts(compose, env)
# logger.info(f"Final compose: {compose}")
content = yaml.safe_dump(compose)
expected_services = compose.get("services", {})
diffs = []
ok = []
for svc_name, svc in expected_services.items():
running_service = current_services.get(svc_name)
if isinstance(running_service, HasVersion):
expected_version = svc.get("labels", {}).get("org.brencher.version")
running_version = running_service.version if running_service is not None else None
l = {
"service": svc_name,
"expected_version": expected_version,
"actual_version": running_version,
"stack": self.stack_name,
}
logger.info(l)
if not running_version or not expected_version:
diffs.append(l)
elif running_version != expected_version:
diffs.append(l)
else:
ok.append(l)
elif isinstance(running_service, HasImage):
expected_image = svc.get("image")
running_image = running_service.image if running_service is not None else None
l = {
"service": svc_name,
"expected_image": expected_image,
"actual_image": running_image,
"stack": self.stack_name,
}
logger.info(l)
if not running_image or not expected_image:
diffs.append(l)
elif running_image != expected_image:
diffs.append(l)
else:
ok.append(l)
else:
diffs.append({
"service": svc_name,
"stack": self.stack_name,
})
if len(diffs) == 0:
logger.info(f"No diff found, stack is already up-to-date.")
return ok
logger.info(f"Diff {diffs}")
if self.env.dry:
logger.info(f"Stack is not active, skipping deploy.")
return {
"diffs": diffs,
}
# tmp_compose_path = os.path.join(tempfile.gettempdir(), f"{hashlib.sha1(content.encode()).hexdigest()[:5]}")
tmp_compose_path = docker_compose_absolute_path + ".tmp"
with open(tmp_compose_path, 'w') as f:
f.write(content)
logger.info(f"Deploying stack '{self.stack_name}' using {tmp_compose_path}")
# Deploy stack to Docker Swarm
cmd = [
"docker", "stack", "deploy",
"--detach=false",
"--prune",
"-c", tmp_compose_path,
self.stack_name
]
logger.info(
f"Deploying stack '{self.stack_name}' using {tmp_compose_path} in cwd {os.path.dirname(tmp_compose_path)}")
swarmEnv: dict[str, str] = {}
if os.path.exists(os.path.join(os.path.dirname(tmp_compose_path), ".env")):
swarmEnv = {k: v for k, v in dotenv_values(os.path.join(os.path.dirname(tmp_compose_path), ".env")).items()
if v is not None}
# merge_dicts(swarmEnv, env)
result = subprocess.run(cmd, capture_output=True, text=True, cwd=os.path.dirname(tmp_compose_path))
#, env=swarmEnv)
os.remove(tmp_compose_path)
logger.info(f"docker stack deploy stdout: {result.stdout or '<empty>'}")
logger.info(f"docker stack deploy stderr: {result.stderr or '<empty>'}")
stderr_lower = result.stderr.lower()
docker_error_markers = (
"failed to ",
"error response from daemon",
"rpc error",
"denied",
"no such image",
)
if result.returncode != 0 or any(marker in stderr_lower for marker in docker_error_markers):
deploy_output = (
f"exit code: {result.returncode}\n"
f"stdout:\n{result.stdout or '<empty>'}\n"
f"stderr:\n{result.stderr or '<empty>'}"
)
logger.error(f"Stack deploy failed:\n{deploy_output}")
raise RuntimeError(f"Stack deploy failed:\n{deploy_output}")
logger.info(f"Stack deployed successfully: {result.stdout}")
# Check health of all deployed services
client = docker.DockerClient(base_url='unix://var/run/docker.sock')
not_running = []
for svc_name in expected_services:
full_svc_name = f"{self.stack_name}_{svc_name}"
services_list = client.services.list(filters={"name": full_svc_name})
if not services_list:
not_running.append(f"{full_svc_name} (not found)")
continue
svc = services_list[0]
spec_mode = svc.attrs.get("Spec", {}).get("Mode", {})
desired_replicas = spec_mode.get("Replicated", {}).get("Replicas", 1)
running_tasks = [
t for t in svc.tasks()
if t.get("Status", {}).get("State") == "running" and t.get("DesiredState") == "running"
]
if len(running_tasks) < desired_replicas:
not_running.append(f"{full_svc_name} ({len(running_tasks)}/{desired_replicas} replicas running)")
if not_running:
raise BaseException(f"Services not started yet: {', '.join(not_running)}")
version_mismatches = []
for svc_name, svc in expected_services.items():
full_svc_name = f"{self.stack_name}_{svc_name}"
services_list = client.services.list(filters={"name": full_svc_name})
if not services_list:
continue
service_labels = services_list[0].attrs.get("Spec", {}).get("TaskTemplate", {}).get("ContainerSpec", {}).get("Labels", {})
expected_version = svc.get("labels", {}).get("org.brencher.version")
actual_version = service_labels.get("org.brencher.version")
if expected_version != actual_version:
version_mismatches.append(
f"{full_svc_name} expected version {expected_version}, got {actual_version}"
)
update_status = services_list[0].attrs.get("UpdateStatus", {})
if update_status.get("State") in {"paused", "rollback_paused", "rollback_started"}:
version_mismatches.append(
f"{full_svc_name} Swarm update {update_status.get('State')}: {update_status.get('Message', '')}"
)
if version_mismatches:
raise RuntimeError(f"Deployment did not apply requested versions: {'; '.join(version_mismatches)}")
return f"Stack deployed successfully: {result.stdout}"