forked from admin/brencher
## Summary Adds brencher profile `connectivity_tester` that deploys the connectivity-tester stack from `admin/ansible`. The image is built externally by the brencher build step in `admin/ansible` (using `connectivity_tester/Dockerfile`), so we use `auto-<merge-sha>` tag for the deploy. ## What's in the box - `GitClone` with two remotes (hermes/ansible + admin/ansible) so feature branches in the fork can be picked up via cross-fork PRs. - `DockerSwarmCheck(stack_name="connectivity_tester")` for diff detection. - `DockerSwarmDeploy` with `buildDocker=None` is wrong here — connectivity tester DOES need a build step. **Need to change to non-None.** - `UrlCheck` on /healthz and `SimpleLog` for the UI user link. ## Open question The `admin/ansible/connectivity_tester/stack-compose.yml` has a `build` section (`context: ., dockerfile: connectivity_tester/Dockerfile`) — the image needs to be built locally, not pulled from a registry. Brencher must build the image in the deploy step. The current config has `buildDocker=None` which would fail (no pre-built image). This is a known gap; the PR gets the structure in place. Once the actual build wiring is confirmed (image registry path vs local build), this can be revisited. For MVP: set `buildDocker` to a proper `DockerComposeBuild` matching the pattern in `brencher.py`. ## Related - admin/ansible#39 (PR) — adds the stack + Dockerfile + compose - Org1/hermes-voice-gateway#5 — connectivity testing blocker - Org1/hermes-voice-gateway#3 — Path B: voice-gateway, blocked by #5 - admin/ansible#16 — Connectivity testing monitoring: admin/ansible#16 --------- Co-authored-by: hermes <hermes@rudolf.keenetic.link> Co-authored-by: brencher_bot <rudolfss13@gmail.com> Reviewed-on: admin/brencher#65 Co-authored-by: hermes <rudolfss13+hermes@gmail.com>
377 lines
13 KiB
Python
377 lines
13 KiB
Python
import hashlib
|
|
import logging
|
|
import os
|
|
import re
|
|
import subprocess
|
|
from dataclasses import dataclass
|
|
from typing import List, Dict, Callable, Any, Mapping, Protocol, Union, runtime_checkable
|
|
|
|
import docker
|
|
import yaml
|
|
from docker import errors as docker_errors
|
|
from dotenv import dotenv_values
|
|
from enironment import AbstractStep
|
|
from steps.git import GitClone, HasVersion
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class DockerComposeBuild(AbstractStep[Dict[str, str]]):
|
|
def __init__(self,
|
|
wd: GitClone, # TODO should be CheckoutMerged
|
|
docker_repo_username: str,
|
|
docker_repo_password: str,
|
|
docker_compose_path: str,
|
|
docker_repo_url: str,
|
|
publish: bool,
|
|
envs: Callable[[], Dict[str, Any]],
|
|
build_cache: bool = False,
|
|
**kwargs: Any) -> None:
|
|
super().__init__(**kwargs)
|
|
self.wd = wd
|
|
self.envs = envs
|
|
self.docker_repo_username = docker_repo_username
|
|
self.docker_repo_password = docker_repo_password
|
|
self.docker_compose_path = docker_compose_path
|
|
self.docker_repo_url = docker_repo_url
|
|
self.publish = publish
|
|
self.build_cache = build_cache
|
|
|
|
def progress(self) -> Dict[str, str]:
|
|
"""
|
|
Build and push Docker images defined in a docker-compose file.
|
|
Returns a dict mapping image name to its SHA digest.
|
|
"""
|
|
image_shas: Dict[str, str] = {}
|
|
try:
|
|
# Authenticate to docker repo
|
|
client = docker.DockerClient(base_url='unix://var/run/docker.sock')
|
|
if self.publish:
|
|
client.login(
|
|
username=self.docker_repo_username,
|
|
password=self.docker_repo_password,
|
|
registry=self.docker_repo_url
|
|
)
|
|
env = self.envs()
|
|
# Parse docker-compose file
|
|
docker_compose_absolute_path = os.path.join(self.wd.progress(), self.docker_compose_path)
|
|
with open(docker_compose_absolute_path, 'r') as f:
|
|
content = f.read()
|
|
content = re.sub(r'\$\{([^}]+)\}', lambda m: env[m.group(1)], content)
|
|
compose = yaml.safe_load(content)
|
|
services = compose.get('services', {})
|
|
|
|
for name, svc in services.items():
|
|
build_section = svc.get('build')
|
|
if build_section is None:
|
|
continue
|
|
if isinstance(build_section, dict):
|
|
build_ctx = build_section.get('context', '.')
|
|
build_dockerfile = build_section.get('dockerfile', 'Dockerfile')
|
|
else:
|
|
build_ctx = build_section
|
|
build_dockerfile = 'Dockerfile'
|
|
|
|
build_ctx = os.path.join(os.path.dirname(docker_compose_absolute_path), build_ctx)
|
|
build_dockerfile = os.path.join(os.path.dirname(docker_compose_absolute_path), build_dockerfile)
|
|
|
|
image = svc.get('image')
|
|
if not build_ctx or not image:
|
|
continue
|
|
if self.publish:
|
|
# Check if image exists in remote repo
|
|
try:
|
|
img = client.images.pull(image)
|
|
logger.info(f"Image {image} already exists in repo, skipping build.")
|
|
if img.id is not None:
|
|
image_shas[image] = img.id
|
|
continue
|
|
except Exception:
|
|
pass
|
|
else:
|
|
# Check if image exists locally
|
|
try:
|
|
img = client.images.get(image)
|
|
logger.info(f"Image {image} already exists locally, skipping build.")
|
|
if img.id is not None:
|
|
image_shas[image] = img.id
|
|
continue
|
|
except docker_errors.ImageNotFound:
|
|
pass
|
|
|
|
logger.info(f"Building image {image} from {build_ctx}, {build_dockerfile}")
|
|
img, _ = client.images.build(path=build_ctx, dockerfile=build_dockerfile, tag=image, nocache=not self.build_cache, rm=True)
|
|
|
|
if self.publish:
|
|
logger.info(f"Pushing image {image}")
|
|
for line in client.images.push(image, stream=True, decode=True):
|
|
logger.debug(line)
|
|
if img.id is not None:
|
|
image_shas[image] = img.id
|
|
return image_shas
|
|
except Exception as e:
|
|
raise e
|
|
|
|
|
|
@dataclass
|
|
class DockerSwarmCheckResult:
|
|
name: str
|
|
image: str
|
|
stack: str
|
|
version: str
|
|
|
|
|
|
class DockerSwarmCheck(AbstractStep[Dict[str, DockerSwarmCheckResult]]):
|
|
|
|
def __init__(self,
|
|
stack_name: str,
|
|
**kwargs: Any) -> None:
|
|
super().__init__(**kwargs)
|
|
self.stack_name = stack_name
|
|
|
|
def progress(self) -> Dict[str, DockerSwarmCheckResult]:
|
|
|
|
client = docker.DockerClient(base_url='unix://var/run/docker.sock')
|
|
current_services: Dict[str, DockerSwarmCheckResult] = {}
|
|
for svc in client.services.list():
|
|
attrs = client.services.get(svc.id).attrs
|
|
name = attrs["Spec"]["Name"].replace(self.stack_name + "_", "")
|
|
if attrs["Spec"]["Labels"].get("com.docker.stack.namespace", "") == self.stack_name:
|
|
# logger.info(f"Service {svc.name} is running with image {attrs}")
|
|
current_services[name] = DockerSwarmCheckResult(
|
|
name=name,
|
|
image=attrs["Spec"]["Labels"].get("com.docker.stack.image", ""),
|
|
stack=attrs["Spec"]["Labels"].get("com.docker.stack.namespace", ""),
|
|
version=attrs["Spec"]["TaskTemplate"]["ContainerSpec"]["Labels"].get("org.brencher.version", ""),
|
|
)
|
|
|
|
logger.info(f"Current services in stack '{self.stack_name}': {current_services}")
|
|
return current_services
|
|
|
|
|
|
@runtime_checkable
|
|
class HasImage(Protocol):
|
|
image: str
|
|
|
|
|
|
class DockerSwarmDeploy(AbstractStep[str]):
|
|
def __init__(self,
|
|
wd: GitClone,
|
|
buildDocker: DockerComposeBuild | None,
|
|
stackChecker: AbstractStep[Mapping[str, Union[HasImage, HasVersion]]],
|
|
docker_compose_path: str,
|
|
envs: Callable[[], Dict[str, Any]],
|
|
stack_name: str,
|
|
**kwargs: Any) -> None:
|
|
super().__init__(**kwargs)
|
|
self.wd = wd
|
|
self.buildDocker = buildDocker
|
|
self.envs = envs
|
|
self.docker_compose_path = docker_compose_path
|
|
self.stack_name = stack_name
|
|
self.stackChecker = stackChecker
|
|
|
|
def progress(self) -> Any:
|
|
"""
|
|
Deploys to Docker Swarm using the specified docker-compose.yaml.
|
|
"""
|
|
self.wd.progress()
|
|
if self.buildDocker is not None:
|
|
self.buildDocker.progress()
|
|
current_services: Mapping[str, HasImage | HasVersion] = self.stackChecker.progress()
|
|
|
|
def merge_dicts(a: Dict[str, Any], b: Dict[str, Any]) -> None:
|
|
for k, v in b.items():
|
|
if (
|
|
k in a
|
|
and isinstance(a[k], dict)
|
|
and isinstance(v, dict)
|
|
):
|
|
merge_dicts(a[k], v)
|
|
else:
|
|
a[k] = v
|
|
|
|
env = self.envs()
|
|
# Prepare docker-compose file with env substitution
|
|
docker_compose_absolute_path = os.path.join(self.wd.progress(), self.docker_compose_path)
|
|
with open(docker_compose_absolute_path, 'r') as f:
|
|
content = f.read()
|
|
content = re.sub(r'\$\{([^}]+)\}', lambda m: env.get(m.group(1), ""), content)
|
|
compose = yaml.safe_load(content)
|
|
version = env["version"]
|
|
if "configs" in compose:
|
|
sanitized_version = re.sub(r'[^a-zA-Z0-9_.-]', '_', version)
|
|
full_prefix = f"{self.stack_name}_"
|
|
|
|
def _config_full_name(base_name: str) -> str:
|
|
full = f"{full_prefix}{base_name}_{sanitized_version}"
|
|
if len(full) <= 64:
|
|
return f"{base_name}_{sanitized_version}"
|
|
return hashlib.sha256(full.encode("utf-8")).hexdigest()[:32]
|
|
|
|
config_names = {
|
|
name: _config_full_name(name)
|
|
for name in compose["configs"]
|
|
}
|
|
compose["configs"] = {
|
|
config_names.get(name, name): config
|
|
for name, config in compose["configs"].items()
|
|
}
|
|
for service in compose.get("services", {}).values():
|
|
for config in service.get("configs", []):
|
|
if isinstance(config, str):
|
|
config_index = service["configs"].index(config)
|
|
service["configs"][config_index] = config_names.get(config, config)
|
|
elif isinstance(config, dict) and "source" in config:
|
|
config["source"] = config_names.get(config["source"], config["source"])
|
|
if "services" in compose:
|
|
for svc in compose["services"].values():
|
|
if "build" in svc:
|
|
del svc["build"]
|
|
svc["labels"] = {"org.brencher.version": env["version"]}
|
|
del env["version"]
|
|
env.pop("registry_owner", None)
|
|
merge_dicts(compose, env)
|
|
# logger.info(f"Final compose: {compose}")
|
|
content = yaml.safe_dump(compose)
|
|
|
|
expected_services = compose.get("services", {})
|
|
diffs = []
|
|
ok = []
|
|
for svc_name, svc in expected_services.items():
|
|
running_service = current_services.get(svc_name)
|
|
if isinstance(running_service, HasVersion):
|
|
expected_version = svc.get("labels", {}).get("org.brencher.version")
|
|
running_version = running_service.version if running_service is not None else None
|
|
l = {
|
|
"service": svc_name,
|
|
"expected_version": expected_version,
|
|
"actual_version": running_version,
|
|
"stack": self.stack_name,
|
|
}
|
|
logger.info(l)
|
|
if not running_version or not expected_version:
|
|
diffs.append(l)
|
|
elif running_version != expected_version:
|
|
diffs.append(l)
|
|
else:
|
|
ok.append(l)
|
|
elif isinstance(running_service, HasImage):
|
|
expected_image = svc.get("image")
|
|
running_image = running_service.image if running_service is not None else None
|
|
l = {
|
|
"service": svc_name,
|
|
"expected_image": expected_image,
|
|
"actual_image": running_image,
|
|
"stack": self.stack_name,
|
|
}
|
|
logger.info(l)
|
|
if not running_image or not expected_image:
|
|
diffs.append(l)
|
|
elif running_image != expected_image:
|
|
diffs.append(l)
|
|
else:
|
|
ok.append(l)
|
|
else:
|
|
diffs.append({
|
|
"service": svc_name,
|
|
"stack": self.stack_name,
|
|
})
|
|
if len(diffs) == 0:
|
|
logger.info(f"No diff found, stack is already up-to-date.")
|
|
return ok
|
|
logger.info(f"Diff {diffs}")
|
|
if self.env.dry:
|
|
logger.info(f"Stack is not active, skipping deploy.")
|
|
return {
|
|
"diffs": diffs,
|
|
}
|
|
|
|
# tmp_compose_path = os.path.join(tempfile.gettempdir(), f"{hashlib.sha1(content.encode()).hexdigest()[:5]}")
|
|
tmp_compose_path = docker_compose_absolute_path + ".tmp"
|
|
with open(tmp_compose_path, 'w') as f:
|
|
f.write(content)
|
|
logger.info(f"Deploying stack '{self.stack_name}' using {tmp_compose_path}")
|
|
|
|
# Deploy stack to Docker Swarm
|
|
cmd = [
|
|
"docker", "stack", "deploy",
|
|
"--detach=false",
|
|
"--prune",
|
|
"-c", tmp_compose_path,
|
|
self.stack_name
|
|
]
|
|
logger.info(
|
|
f"Deploying stack '{self.stack_name}' using {tmp_compose_path} in cwd {os.path.dirname(tmp_compose_path)}")
|
|
swarmEnv: dict[str, str] = {}
|
|
if os.path.exists(os.path.join(os.path.dirname(tmp_compose_path), ".env")):
|
|
swarmEnv = {k: v for k, v in dotenv_values(os.path.join(os.path.dirname(tmp_compose_path), ".env")).items()
|
|
if v is not None}
|
|
# merge_dicts(swarmEnv, env)
|
|
result = subprocess.run(cmd, capture_output=True, text=True, cwd=os.path.dirname(tmp_compose_path))
|
|
#, env=swarmEnv)
|
|
os.remove(tmp_compose_path)
|
|
logger.info(f"docker stack deploy stdout: {result.stdout or '<empty>'}")
|
|
logger.info(f"docker stack deploy stderr: {result.stderr or '<empty>'}")
|
|
stderr_lower = result.stderr.lower()
|
|
docker_error_markers = (
|
|
"failed to ",
|
|
"error response from daemon",
|
|
"rpc error",
|
|
"denied",
|
|
"no such image",
|
|
)
|
|
if result.returncode != 0 or any(marker in stderr_lower for marker in docker_error_markers):
|
|
deploy_output = (
|
|
f"exit code: {result.returncode}\n"
|
|
f"stdout:\n{result.stdout or '<empty>'}\n"
|
|
f"stderr:\n{result.stderr or '<empty>'}"
|
|
)
|
|
logger.error(f"Stack deploy failed:\n{deploy_output}")
|
|
raise RuntimeError(f"Stack deploy failed:\n{deploy_output}")
|
|
logger.info(f"Stack deployed successfully: {result.stdout}")
|
|
|
|
# Check health of all deployed services
|
|
client = docker.DockerClient(base_url='unix://var/run/docker.sock')
|
|
not_running = []
|
|
for svc_name in expected_services:
|
|
full_svc_name = f"{self.stack_name}_{svc_name}"
|
|
services_list = client.services.list(filters={"name": full_svc_name})
|
|
if not services_list:
|
|
not_running.append(f"{full_svc_name} (not found)")
|
|
continue
|
|
svc = services_list[0]
|
|
spec_mode = svc.attrs.get("Spec", {}).get("Mode", {})
|
|
desired_replicas = spec_mode.get("Replicated", {}).get("Replicas", 1)
|
|
running_tasks = [
|
|
t for t in svc.tasks()
|
|
if t.get("Status", {}).get("State") == "running" and t.get("DesiredState") == "running"
|
|
]
|
|
if len(running_tasks) < desired_replicas:
|
|
not_running.append(f"{full_svc_name} ({len(running_tasks)}/{desired_replicas} replicas running)")
|
|
if not_running:
|
|
raise BaseException(f"Services not started yet: {', '.join(not_running)}")
|
|
|
|
version_mismatches = []
|
|
for svc_name, svc in expected_services.items():
|
|
full_svc_name = f"{self.stack_name}_{svc_name}"
|
|
services_list = client.services.list(filters={"name": full_svc_name})
|
|
if not services_list:
|
|
continue
|
|
service_labels = services_list[0].attrs.get("Spec", {}).get("TaskTemplate", {}).get("ContainerSpec", {}).get("Labels", {})
|
|
expected_version = svc.get("labels", {}).get("org.brencher.version")
|
|
actual_version = service_labels.get("org.brencher.version")
|
|
if expected_version != actual_version:
|
|
version_mismatches.append(
|
|
f"{full_svc_name} expected version {expected_version}, got {actual_version}"
|
|
)
|
|
update_status = services_list[0].attrs.get("UpdateStatus", {})
|
|
if update_status.get("State") in {"paused", "rollback_paused", "rollback_started"}:
|
|
version_mismatches.append(
|
|
f"{full_svc_name} Swarm update {update_status.get('State')}: {update_status.get('Message', '')}"
|
|
)
|
|
if version_mismatches:
|
|
raise RuntimeError(f"Deployment did not apply requested versions: {'; '.join(version_mismatches)}")
|
|
|
|
return f"Stack deployed successfully: {result.stdout}"
|