Deployfix (#22)

* fix

* Comment out merge_dicts call in DockerSwarmDeploy to prevent merging environment variables

* Comment out cwd and env parameters in subprocess.run for DockerSwarmDeploy to simplify execution

* Replace environment variable placeholders in docker-compose file with actual values before loading

* Add environment variables for brencher-backend in Docker configuration

* add conflict for testing

* Add extra_hosts configuration for brencher-backend service in Docker Compose

* Update SLAVE_BRENCHER environment variable to use host.docker.internal for better compatibility

* Improve remote connection handling for SLAVE_BRENCHER with retry logic

* Remove 'brencher' from PROFILES environment variable in Docker configuration

* Refactor Docker Compose configuration by removing inline SLAVE_BRENCHER environment variable and ensuring consistency in service definitions.

* test change

* Fix emit_envs function to use get_global_envs_to_emit for environment updates

* Refactor merge_dicts function to improve key merging logic and ensure correct handling of dictionary values.

* Add branches_slaves to manage slave branch data and update event handlers

* Refactor branch handling by removing redundant payload marking and ensuring correct merging of branches and slave branches.
This commit is contained in:
2025-11-02 01:01:54 +01:00
committed by GitHub
parent f2eb17f1f7
commit 8b4d0e03fa
4 changed files with 41 additions and 38 deletions
+28 -26
View File
@@ -64,6 +64,7 @@ FRONTEND_DIR = os.path.join(os.path.dirname(__file__), '../frontend')
environments: Dict[str, Tuple[enironment.Environment, List[AbstractStep]]] = {}
environments_slaves: Dict[str, Tuple[enironment.Environment, List[AbstractStep]]] = {}
branches = {}
branches_slaves = {}
state_lock = threading.Lock()
@app.route('/')
@@ -81,15 +82,17 @@ from flask_socketio import Namespace
T = TypeVar('T')
def merge_dicts(a: Dict[str, T], b: Dict[str, T]) -> Dict[str, T]:
result: Dict[str, T] = {}
for k, v in b.items():
for k in (a.keys() | b.keys()):
if k not in a or k not in b:
result[k] = a.get(k, b.get(k)) # type: ignore[arg-type]
continue
if (
k in a
and isinstance(a[k], dict)
and isinstance(v, dict)
isinstance(a[k], dict)
and isinstance(b[k], dict)
):
result[k] = merge_dicts(a[k], v) # type: ignore[arg-type]
result[k] = merge_dicts(a[k], b[k]) # type: ignore[arg-type]
else:
result[k] = v
result[k] = b[k]
return result
slave_url = os.getenv('SLAVE_BRENCHER')
@@ -98,18 +101,14 @@ if slave_url:
logger.info(f"SLAVE_BRENCHER set, will connect to master at {slave_url}")
remote_sio = socketio_client.Client(logger=False, engineio_logger=False)
def _mark_forwarded(payload):
if isinstance(payload, dict):
return {**payload, '_slave_forwarded': True}
return {'_slave_forwarded': True, '__payload': payload}
# Handlers for events from master -> re-emit locally (marked so we don't loop)
@remote_sio.on('branches', namespace='/ws/branches')
def _remote_branches(data):
global branches
merge_result = merge_dicts(branches, data)
global branches, branches_slaves
branches_slaves = data
merge_result = merge_dicts(branches, branches_slaves)
try:
socketio.emit('branches', _mark_forwarded(merge_result), namespace='/ws/branches')
socketio.emit('branches', merge_result, namespace='/ws/branches')
except Exception as e:
logger.error(f"Error forwarding remote branches locally: {e}")
@@ -125,7 +124,7 @@ if slave_url:
@remote_sio.on('error', namespace='/ws/errors')
def _remote_error(data):
try:
socketio.emit('error', _mark_forwarded(data), namespace='/ws/errors')
socketio.emit('error', data, namespace='/ws/errors')
except Exception as e:
logger.error(f"Error forwarding remote errors locally: {e}")
@@ -140,11 +139,11 @@ if slave_url:
class BranchesNamespace(Namespace):
def on_connect(self, auth=None):
emit('branches', branches)
emit('branches', merge_dicts(branches, branches_slaves))
def on_update(self, data):
emit('branches', branches)
emit('branches', merge_dicts(branches, branches_slaves))
def get_local_envs_to_emit():
def get_local_envs_to_emit() -> Dict[str, Tuple[enironment.Environment, List[AbstractStep]]]:
env_dtos = {}
for e, p in environments.values():
env = asdict(e)
@@ -256,7 +255,7 @@ if __name__ == '__main__':
# Background thread to refresh branches every 5 minutes
def emit_fresh_branches():
global branches, environments
global branches, branches_slaves, environments
for k, e in environments.items():
env, pipe = e
branches[k] = {}
@@ -271,7 +270,7 @@ if __name__ == '__main__':
socketio.emit('error', {'message': e}, namespace='/ws/errors')
logger.info(f"Fetched {env.id}: {len (branches[env.id])} branches")
# logger.info(f"Fetched {env.id}: {branches[env.id]}")
socketio.emit('branches', branches, namespace='/ws/branches')
socketio.emit('branches', merge_dicts(branches, branches_slaves), namespace='/ws/branches')
def processing_thread():
@@ -279,7 +278,7 @@ if __name__ == '__main__':
import processing
def emit_envs():
try:
socketio.emit('environments', get_local_envs_to_emit(), namespace='/ws/environment')
socketio.emit('environments', get_global_envs_to_emit(), namespace='/ws/environment')
except Exception as e:
logger.error(f"Error emitting environments: {str(e)}")
with state_lock:
@@ -289,6 +288,7 @@ if __name__ == '__main__':
environment_update_event.wait(timeout=1*60)
environment_update_event.clear()
processing = threading.Thread(target=processing_thread)
processing.daemon = True
processing.start()
@@ -298,11 +298,13 @@ if __name__ == '__main__':
def _connect_remote():
global remote_sio
print("Connecting to SLAVE_BRENCHER...")
try:
if remote_sio is not None:
remote_sio.connect(slave_url, namespaces=['/ws/branches', '/ws/environment', '/ws/errors'])
except Exception as e:
logger.error(f"Could not connect to SLAVE_BRENCHER {slave_url}: {e}")
while True:
try:
if remote_sio is not None and not remote_sio.connected:
remote_sio.connect(slave_url, namespaces=['/ws/branches', '/ws/environment', '/ws/errors'])
except Exception as e:
logger.error(f"Could not connect to SLAVE_BRENCHER {slave_url}: {e}")
time.sleep(60)
t = threading.Thread(target=_connect_remote, daemon=True)
t.start()
-8
View File
@@ -44,14 +44,6 @@ def create_pipeline(env: Environment) -> List[AbstractStep]:
stackChecker=dockerSwarmCheck,
envs = lambda: {
"version": "auto-" + checkoutMerged.result.version,
"services": {
"brencher-backend" :{
"environment": {
"SLAVE_BRENCHER" : "http://192.169.1.96:5002"
},
}
}
# SLAVE_BRENCHER='http://100.70.193.97:5002' .venv/bin/python3 backend/app.py brencher_localX
},
stack_name = "brencher",
docker_compose_path = "docker-compose.yml",
+9 -4
View File
@@ -165,7 +165,9 @@ class DockerSwarmDeploy(AbstractStep[str]):
# Prepare docker-compose file with env substitution
docker_compose_absolute_path = os.path.join(self.wd.result, self.docker_compose_path)
with open(docker_compose_absolute_path, 'r') as f:
compose = yaml.safe_load(f.read())
content = f.read()
content = re.sub(r'\$\{([^}]+)\}', lambda m: env.get(m.group(1), ""), content)
compose = yaml.safe_load(content)
if "services" in compose:
for svc in compose["services"].values():
if "build" in svc:
@@ -218,9 +220,12 @@ class DockerSwarmDeploy(AbstractStep[str]):
self.stack_name
]
logger.info(f"Deploying stack '{self.stack_name}' using {tmp_compose_path} in cwd {os.path.dirname(tmp_compose_path)}")
swarmEnv: dict[str, str] = { k:v for k,v in dotenv_values(os.path.join(os.path.dirname(tmp_compose_path), ".env")).items() if v is not None }
merge_dicts(swarmEnv, env)
result = subprocess.run(cmd, capture_output=True, text=True, cwd=os.path.dirname(tmp_compose_path), env=swarmEnv)
swarmEnv: dict[str, str] = {}
if os.path.exists(os.path.join(os.path.dirname(tmp_compose_path), ".env")):
swarmEnv = { k:v for k,v in dotenv_values(os.path.join(os.path.dirname(tmp_compose_path), ".env")).items() if v is not None }
# merge_dicts(swarmEnv, env)
result = subprocess.run(cmd, capture_output=True, text=True)
#, cwd=os.path.dirname(tmp_compose_path), env=swarmEnv)
if result.returncode != 0:
logger.error(f"Stack deploy failed: {result.stderr}")
raise RuntimeError(f"Stack deploy failed: {result.stderr}")
+4
View File
@@ -4,6 +4,8 @@ services:
build: .
image: registry.rudolf.keenetic.link/brencher:${version}
user: 1000:137
extra_hosts:
- "host.docker.internal:host-gateway"
ports:
- "5001:5001"
volumes:
@@ -15,6 +17,8 @@ services:
- VIRTUAL_HOST=brencher.rudolf.keenetic.link
- LETSENCRYPT_HOST=brencher.rudolf.keenetic.link
- PROFILES=brencher2,torrserv_proxy,immich
- SLAVE_BRENCHER=http://host.docker.internal:5002
# - SLAVE_BRENCHER='http://100.70.193.97:5002' .venv/bin/python3 backend/app.py brencher_localX
deploy:
replicas: 1
restart_policy: