+2
-1
@@ -3,4 +3,5 @@ __pycache__
|
||||
local.env
|
||||
temp
|
||||
venv
|
||||
.venv
|
||||
.venv
|
||||
LOCAL.md
|
||||
@@ -15,3 +15,20 @@ App should be delivered as docker image.
|
||||
|
||||
Generate docker compose file to deploy to docker swarm.
|
||||
|
||||
# Update docker secret
|
||||
|
||||
|
||||
|
||||
docker service scale brencher_brencher-backend=0 brencher2_brencher-backend=0
|
||||
|
||||
docker service update --secret-rm brencher-secrets brencher_brencher-backend
|
||||
docker service update --secret-rm brencher-secrets brencher2_brencher-backend
|
||||
|
||||
docker secret rm brencher-secrets
|
||||
|
||||
printf "GIT_USERNAME = git\nGIT_PASSWORD = \"TODO\"\n" | docker secret create brencher-secrets_new -
|
||||
|
||||
docker service update --secret-add brencher-secrets brencher_brencher-backend
|
||||
docker service update --secret-add brencher-secrets brencher2_brencher-backend
|
||||
|
||||
docker service scale brencher_brencher-backend=1 brencher2_brencher-backend=1
|
||||
|
||||
+23
-10
@@ -19,6 +19,7 @@ from typing import List, Dict, Any, Optional, Tuple
|
||||
import shutil
|
||||
import subprocess
|
||||
import traceback
|
||||
from typing import TypeVar
|
||||
|
||||
# Configure logging
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
||||
@@ -77,16 +78,19 @@ def serve_static(path):
|
||||
from flask import copy_current_request_context
|
||||
from flask_socketio import Namespace
|
||||
|
||||
def merge_dicts(a, b):
|
||||
T = TypeVar('T')
|
||||
def merge_dicts(a: Dict[str, T], b: Dict[str, T]) -> Dict[str, T]:
|
||||
result: Dict[str, T] = {}
|
||||
for k, v in b.items():
|
||||
if (
|
||||
k in a
|
||||
and isinstance(a[k], dict)
|
||||
and isinstance(v, dict)
|
||||
):
|
||||
merge_dicts(a[k], v)
|
||||
result[k] = merge_dicts(a[k], v) # type: ignore[arg-type]
|
||||
else:
|
||||
a[k] = v
|
||||
result[k] = v
|
||||
return result
|
||||
|
||||
slave_url = os.getenv('SLAVE_BRENCHER')
|
||||
remote_sio: socketio_client.Client | None = None
|
||||
@@ -103,9 +107,9 @@ if slave_url:
|
||||
@remote_sio.on('branches', namespace='/ws/branches')
|
||||
def _remote_branches(data):
|
||||
global branches
|
||||
merge_dicts(branches, data)
|
||||
merge_result = merge_dicts(branches, data)
|
||||
try:
|
||||
socketio.emit('branches', _mark_forwarded(data), namespace='/ws/branches')
|
||||
socketio.emit('branches', _mark_forwarded(merge_result), namespace='/ws/branches')
|
||||
except Exception as e:
|
||||
logger.error(f"Error forwarding remote branches locally: {e}")
|
||||
|
||||
@@ -162,9 +166,14 @@ def get_local_envs_to_emit():
|
||||
|
||||
def get_global_envs_to_emit():
|
||||
global environments, environments_slaves
|
||||
data_to_submit = get_local_envs_to_emit()
|
||||
merge_dicts(data_to_submit, environments_slaves)
|
||||
return data_to_submit
|
||||
local_envs = get_local_envs_to_emit()
|
||||
merge_result = merge_dicts(local_envs, environments_slaves)
|
||||
logger.info(f"Local keys: {local_envs.keys()}")
|
||||
logger.info(f"Slave keys: {environments_slaves.keys()}")
|
||||
common_keys = set(local_envs.keys()) & set(environments_slaves.keys())
|
||||
if len(common_keys) > 0:
|
||||
socketio.emit('error', {'message': f"Conflict: both master and slave have environment with id {common_keys}"}, namespace='/ws/errors')
|
||||
return merge_result
|
||||
|
||||
environment_update_event = threading.Event()
|
||||
|
||||
@@ -208,12 +217,14 @@ if __name__ == '__main__':
|
||||
import configs.brencher_local2
|
||||
import configs.brencher_local1
|
||||
import configs.torrserv_proxy
|
||||
import configs.immich
|
||||
environments_l = [
|
||||
configs.brencher.brencher,
|
||||
configs.brencher2.brencher,
|
||||
configs.brencher_local2.brencher_local,
|
||||
configs.brencher_local1.brencher_local,
|
||||
configs.torrserv_proxy.config
|
||||
configs.torrserv_proxy.config,
|
||||
configs.immich.config,
|
||||
]
|
||||
environments = {e[0].id: e for e in environments_l}
|
||||
|
||||
@@ -223,15 +234,17 @@ if __name__ == '__main__':
|
||||
cli_env_ids = os.getenv('PROFILES', '')
|
||||
else:
|
||||
cli_env_ids = cli_env_ids[0]
|
||||
|
||||
logger.info(f"cli_env_ids {cli_env_ids}")
|
||||
if len(cli_env_ids) > 0 and cli_env_ids[0] == '-':
|
||||
cli_env_ids = cli_env_ids[1:].split(',')
|
||||
cli_env_ids = [x for x in cli_env_ids if len(x) > 0]
|
||||
logger.info(f"cli_env_ids (minus) {cli_env_ids}")
|
||||
if cli_env_ids and len(cli_env_ids) > 0:
|
||||
environments = { k: e for k, e in environments.items() if k not in cli_env_ids }
|
||||
else:
|
||||
cli_env_ids = cli_env_ids.split(',')
|
||||
cli_env_ids = [x for x in cli_env_ids if len(x) > 0]
|
||||
logger.info(f"cli_env_ids {cli_env_ids}")
|
||||
if cli_env_ids and len(cli_env_ids) > 0:
|
||||
environments = { k: e for k, e in environments.items() if k in cli_env_ids }
|
||||
|
||||
|
||||
@@ -47,7 +47,7 @@ def create_pipeline(env: Environment) -> List[AbstractStep]:
|
||||
"services": {
|
||||
"brencher-backend" :{
|
||||
"environment": {
|
||||
"SLAVE_BRENCHER" : "192.169.1.96:5002"
|
||||
"SLAVE_BRENCHER" : "http://192.169.1.96:5002"
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
from steps.git import GitClone, CheckoutMerged, GitUnmerge
|
||||
from steps.docker import DockerComposeBuild, DockerSwarmCheck, DockerSwarmDeploy
|
||||
from enironment import Environment
|
||||
from typing import List, Dict, Any, Optional, Tuple
|
||||
from steps.step import AbstractStep
|
||||
|
||||
env = Environment(
|
||||
id="immich",
|
||||
branches=[("ansible/master", "HEAD")],
|
||||
dry=False,
|
||||
repo="https://github.com/rudolf1/uber_backup.git",
|
||||
)
|
||||
|
||||
|
||||
def create_pipeline(env: Environment) -> List[AbstractStep]:
|
||||
clone = GitClone(env, branchNamePrefix="ansible")
|
||||
checkoutMerged = CheckoutMerged(clone, env=env,
|
||||
push = False,
|
||||
|
||||
git_user_email="rudolfss13@gmail.com",
|
||||
git_user_name="brencher_bot"
|
||||
)
|
||||
|
||||
|
||||
dockerSwarmCheck = DockerSwarmCheck(
|
||||
stack_name = "immich",
|
||||
env=env,
|
||||
)
|
||||
deployDocker = DockerSwarmDeploy(
|
||||
wd=clone,
|
||||
buildDocker=None,
|
||||
stackChecker=dockerSwarmCheck,
|
||||
envs = lambda: {
|
||||
"version": "auto-" + checkoutMerged.result.version
|
||||
},
|
||||
stack_name = "immich",
|
||||
docker_compose_path = "poc/immich/stack-compose.yml",
|
||||
env=env,
|
||||
)
|
||||
unmerge = GitUnmerge(clone, dockerSwarmCheck, env=env)
|
||||
|
||||
return [
|
||||
clone,
|
||||
checkoutMerged,
|
||||
dockerSwarmCheck,
|
||||
unmerge,
|
||||
deployDocker,
|
||||
]
|
||||
|
||||
config: Tuple[Environment, List[AbstractStep]] = (env, create_pipeline(env))
|
||||
|
||||
+19
-17
@@ -3,6 +3,7 @@ import subprocess
|
||||
import logging
|
||||
import docker
|
||||
import yaml
|
||||
from dotenv import dotenv_values
|
||||
from typing import List, Optional, Dict, Callable, Any
|
||||
from dataclasses import dataclass
|
||||
from steps.step import AbstractStep
|
||||
@@ -123,7 +124,7 @@ class DockerSwarmCheck(AbstractStep[Dict[str, DockerSwarmCheckResult]]):
|
||||
class DockerSwarmDeploy(AbstractStep[str]):
|
||||
def __init__(self,
|
||||
wd: GitClone,
|
||||
buildDocker: DockerComposeBuild,
|
||||
buildDocker: DockerComposeBuild | None,
|
||||
stackChecker: DockerSwarmCheck,
|
||||
docker_compose_path: str,
|
||||
envs: Callable[[], Dict[str, Any]],
|
||||
@@ -144,33 +145,32 @@ class DockerSwarmDeploy(AbstractStep[str]):
|
||||
"""
|
||||
if isinstance(self.wd.result, BaseException):
|
||||
raise self.wd.result
|
||||
if isinstance(self.buildDocker.result, BaseException):
|
||||
if self.buildDocker is not None and isinstance(self.buildDocker.result, BaseException):
|
||||
raise self.buildDocker.result
|
||||
if isinstance(self.stackChecker.result, BaseException):
|
||||
raise self.stackChecker.result
|
||||
|
||||
|
||||
def merge_dicts(a, b):
|
||||
for k, v in b.items():
|
||||
if (
|
||||
k in a
|
||||
and isinstance(a[k], dict)
|
||||
and isinstance(v, dict)
|
||||
):
|
||||
merge_dicts(a[k], v)
|
||||
else:
|
||||
a[k] = v
|
||||
env = self.envs()
|
||||
# Prepare docker-compose file with env substitution
|
||||
docker_compose_absolute_path = os.path.join(self.wd.result, self.docker_compose_path)
|
||||
with open(docker_compose_absolute_path, 'r') as f:
|
||||
content = f.read()
|
||||
content = re.sub(r'\$\{([^}]+)\}', lambda m: env.get(m.group(1), ""), content)
|
||||
compose = yaml.safe_load(content)
|
||||
compose = yaml.safe_load(f.read())
|
||||
if "services" in compose:
|
||||
for svc in compose["services"].values():
|
||||
if "build" in svc:
|
||||
del svc["build"]
|
||||
svc["labels"] = {"org.brencher.version": env["version"]}
|
||||
def merge_dicts(a, b):
|
||||
for k, v in b.items():
|
||||
if (
|
||||
k in a
|
||||
and isinstance(a[k], dict)
|
||||
and isinstance(v, dict)
|
||||
):
|
||||
merge_dicts(a[k], v)
|
||||
else:
|
||||
a[k] = v
|
||||
del env["version"]
|
||||
merge_dicts(compose, env)
|
||||
# logger.info(f"Final compose: {compose}")
|
||||
@@ -217,8 +217,10 @@ class DockerSwarmDeploy(AbstractStep[str]):
|
||||
"-c", tmp_compose_path,
|
||||
self.stack_name
|
||||
]
|
||||
logger.info(f"Deploying stack '{self.stack_name}' using {tmp_compose_path}")
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
logger.info(f"Deploying stack '{self.stack_name}' using {tmp_compose_path} in cwd {os.path.dirname(tmp_compose_path)}")
|
||||
swarmEnv: dict[str, str] = { k:v for k,v in dotenv_values(os.path.join(os.path.dirname(tmp_compose_path), ".env")).items() if v is not None }
|
||||
merge_dicts(swarmEnv, env)
|
||||
result = subprocess.run(cmd, capture_output=True, text=True, cwd=os.path.dirname(tmp_compose_path), env=swarmEnv)
|
||||
if result.returncode != 0:
|
||||
logger.error(f"Stack deploy failed: {result.stderr}")
|
||||
raise RuntimeError(f"Stack deploy failed: {result.stderr}")
|
||||
|
||||
+12
-2
@@ -13,11 +13,21 @@ from enironment import Environment
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class GitClone(AbstractStep[str]):
|
||||
def __init__(self, env: Environment, branchNamePrefix="", **kwargs):
|
||||
def __init__(self, env: Environment, branchNamePrefix="", credEnvPrefix="GIT", **kwargs):
|
||||
super().__init__(env, **kwargs)
|
||||
self.temp_dir = os.path.join(tempfile.gettempdir(), f"{self.env.id}_{hashlib.sha1(self.env.repo.encode()).hexdigest()[:5]}")
|
||||
self.branchNamePrefix = branchNamePrefix
|
||||
self.credEnvPrefix = credEnvPrefix
|
||||
|
||||
def _get_auth_git_url(self, url: str) -> str:
|
||||
username = os.getenv(f'{self.credEnvPrefix}_USERNAME')
|
||||
password = os.getenv(f'{self.credEnvPrefix}_PASSWORD')
|
||||
if username and password:
|
||||
# Extract protocol and the rest of the URL
|
||||
protocol, rest = url.split('://')
|
||||
return f"{protocol}://{username}:{password}@{rest}"
|
||||
|
||||
return url
|
||||
|
||||
def progress(self) -> str:
|
||||
|
||||
@@ -33,7 +43,7 @@ class GitClone(AbstractStep[str]):
|
||||
repo = git.Repo.init(self.temp_dir)
|
||||
repo.remotes.append(repo.create_remote(
|
||||
'origin',
|
||||
self.env.repo
|
||||
self._get_auth_git_url(self.env.repo)
|
||||
))
|
||||
if self.branchNamePrefix != "":
|
||||
repo.config_writer().set_value('remote "origin"',"fetch", f"+refs/heads/{self.branchNamePrefix}/*:refs/remotes/origin/{self.branchNamePrefix}/*").release()
|
||||
|
||||
+1
-1
@@ -14,7 +14,7 @@ services:
|
||||
- PYTHONUNBUFFERED=1
|
||||
- VIRTUAL_HOST=brencher.rudolf.keenetic.link
|
||||
- LETSENCRYPT_HOST=brencher.rudolf.keenetic.link
|
||||
- PROFILES=brencher2,torrserv_proxy
|
||||
- PROFILES=brencher2,torrserv_proxy,immich
|
||||
deploy:
|
||||
replicas: 1
|
||||
restart_policy:
|
||||
|
||||
+34
-2
@@ -242,9 +242,24 @@ function renderJobs() {
|
||||
/(https?:\/\/[^\s"']+)/g,
|
||||
url => `<a href="${url}" target="_blank" rel="noopener noreferrer">${url}</a>`
|
||||
);
|
||||
const key = `${envObj.id}::${job.name}`;
|
||||
const storageKey = 'jobSpoiler:' + key;
|
||||
const safeId = 'spoiler-' + encodeURIComponent(key).replace(/[^a-zA-Z0-9_-]/g, '_');
|
||||
const isError = /error|exception/i.test(statusDisplay);
|
||||
const openByDefault = isError || (window._jobSpoilerState && window._jobSpoilerState[storageKey] === 'open');
|
||||
|
||||
return `
|
||||
<div class="job-item">
|
||||
<strong>${job.env} - ${job.name}</strong><br />${statusDisplay}
|
||||
<div class="job-header" style="cursor:pointer;font-weight:bold;"
|
||||
onclick="toggleJobSpoiler('${key}', '${safeId}')">
|
||||
${isError
|
||||
? `<span style="color:#dc3545;font-weight:bold;margin-right:6px;" title="Error">!</span>`
|
||||
: `<span style="color:#28a745;font-weight:bold;margin-right:6px;" title="OK">✔</span>`}
|
||||
${envObj.id} - ${job.name}
|
||||
</div>
|
||||
<div id="${safeId}" class="job-spoiler" style="display: ${openByDefault ? 'block' : 'none'}; margin-top:8px;">
|
||||
${statusDisplay}
|
||||
</div>
|
||||
</div>`;
|
||||
}).join('')
|
||||
: '<div class="job-item">No jobs found.</div>'}
|
||||
@@ -252,7 +267,24 @@ function renderJobs() {
|
||||
}).join('');
|
||||
}
|
||||
|
||||
|
||||
toggleJobSpoiler = function(key, safeId) {
|
||||
try {
|
||||
const el = document.getElementById(safeId);
|
||||
if (!el) return;
|
||||
const storageKey = 'jobSpoiler:' + key;
|
||||
const isHidden = window.getComputedStyle(el).display === 'none';
|
||||
window._jobSpoilerState = window._jobSpoilerState || {};
|
||||
if (isHidden) {
|
||||
el.style.display = 'block';
|
||||
window._jobSpoilerState[storageKey] = 'open';
|
||||
} else {
|
||||
el.style.display = 'none';
|
||||
window._jobSpoilerState[storageKey] = 'closed';
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('toggleJobSpoiler error', err);
|
||||
}
|
||||
};
|
||||
refreshBranchesBtn.onclick = () => {
|
||||
wsEnv.emit('update', { id: "" });
|
||||
showStatus('Refreshing...');
|
||||
|
||||
Reference in New Issue
Block a user