Torrserv (#11)

* torreserv

* profiles

* fix

* fix

* fix

* fix

* fix

* fix
This commit is contained in:
2025-09-12 12:25:01 +02:00
committed by GitHub
parent f0f4edb3a7
commit fd384e6107
6 changed files with 137 additions and 34 deletions
+12 -23
View File
@@ -14,6 +14,7 @@ from dataclasses import dataclass, asdict, field
from typing import List, Dict, Any, Optional
import shutil
import subprocess
import traceback
# Configure logging
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
@@ -85,9 +86,10 @@ def get_envs_to_emit():
res = []
for r in p:
if isinstance(r.result_obj, BaseException):
stack = traceback.format_exception(type(r.result_obj), r.result_obj, r.result_obj.__traceback__)
res.append({
"name":r.name,
"status": str(r.result_obj)
"name": r.name,
"status": [str(r.result_obj), stack],
})
else:
res.append({
@@ -131,11 +133,13 @@ if __name__ == '__main__':
import configs.brencher
import configs.brencher_local2
import configs.brencher_local1
import configs.torrserv_proxy
environments = [
configs.brencher.brencher,
configs.brencher_local2.brencher_local,
configs.brencher_local1.brencher_local
]
configs.brencher_local1.brencher_local,
configs.torrserv_proxy.config
]
import sys
cli_env_ids = sys.argv[1:]
@@ -170,27 +174,12 @@ if __name__ == '__main__':
try:
if not isinstance(step, GitClone):
continue
repo = git.Repo(step.result)
for ref in repo.refs:
if ref.name.startswith('origin/') and not ref.name.startswith('origin/HEAD'):
branch_name = ref.name [len('origin/'):]
if not branch_name.startswith('auto/'): # Skip auto branches
branches[env.id][branch_name] = []
cmt = [ref.commit]
for _ in range(10):
for commit in cmt:
branches[env.id][branch_name].append({
'hexsha': commit.hexsha,
'author': commit.author.name,
'date': commit.committed_datetime.isoformat(),
'message': commit.message.strip()
})
cmt = [p for c in cmt for p in c.parents]
logger.info(f"Fetched {env.id}: {len (branches[env.id])} branches")
branches[env.id] = {**branches[env.id], **step.get_branches()}
except BaseException as e:
socketio.emit('error', {'message': e}, namespace='/ws/errors')
socketio.emit('branches', branches, namespace='/ws/branches')
logger.info(f"Fetched {env.id}: {len (branches[env.id])} branches")
# logger.info(f"Fetched {env.id}: {branches[env.id]}")
socketio.emit('branches', branches, namespace='/ws/branches')
def processing_thread():
+64
View File
@@ -0,0 +1,64 @@
from steps.git import GitClone, CheckoutMerged, GitUnmerge
from steps.docker import DockerComposeBuild, DockerSwarmCheck, DockerSwarmDeploy
from enironment import Environment
from typing import List, Dict, Any, Optional, Tuple
from steps.step import AbstractStep
env = Environment(
id="torrserv_proxy",
branches=[],
# ("photosHelper/streams", "HEAD")
dry=False,
repo="https://github.com/rudolf1/TorrServer.git",
)
def create_pipeline(env: Environment) -> List[AbstractStep]:
clone = GitClone(env)
checkoutMerged = CheckoutMerged(clone, env=env,
push = False,
git_user_email="rudolfss13@gmail.com",
git_user_name="brencher_bot"
)
buildDocker = DockerComposeBuild(clone,
docker_repo_username = "",
docker_repo_password = "",
docker_compose_path = "reverseproxy/docker-compose.yml",
docker_repo_url="https://registry.rudolf.keenetic.link",
publish=False,
envs = lambda: {
"version": "auto-" + checkoutMerged.result.version,
},
env=env
)
dockerSwarmCheck = DockerSwarmCheck(
stack_name = "torrserv_proxy",
env=env,
)
deployDocker = DockerSwarmDeploy(
wd=clone,
buildDocker=buildDocker,
stackChecker=dockerSwarmCheck,
envs = lambda: {
"version": "auto-" + checkoutMerged.result.version
},
stack_name = "torrserv_proxy",
docker_compose_path = "reverseproxy/docker-compose.yml",
env=env,
)
unmerge = GitUnmerge(clone, dockerSwarmCheck, env=env)
return [
clone,
checkoutMerged,
buildDocker,
dockerSwarmCheck,
unmerge,
deployDocker,
]
config: Tuple[Environment, List[AbstractStep]] = (env, create_pipeline(env))
+1 -1
View File
@@ -50,7 +50,7 @@ class DockerComposeBuild(AbstractStep[List[str]]):
services = compose.get('services', {})
for name, svc in services.items():
build_ctx = os.path.join(self.wd.result, svc.get('build'))
build_ctx = os.path.join(os.path.dirname(docker_compose_absolute_path), svc.get('build'))
image = svc.get('image')
if not build_ctx or not image:
continue
+35 -4
View File
@@ -4,7 +4,7 @@ from git.objects import Commit
from git.refs import Reference
import logging
from dataclasses import dataclass, asdict, field
from typing import List, Optional, Union, Tuple, Set, Iterator, Dict
from typing import List, Optional, Union, Tuple, Set, Iterator, Dict, Any
from steps.step import AbstractStep
import tempfile
import os
@@ -13,9 +13,11 @@ from enironment import Environment
logger = logging.getLogger(__name__)
class GitClone(AbstractStep[str]):
def __init__(self, env: Environment, **kwargs):
def __init__(self, env: Environment, branchNamePrefix="", **kwargs):
super().__init__(env, **kwargs)
self.temp_dir = os.path.join(tempfile.gettempdir(), f"{self.env.id}_{hashlib.sha1(self.env.repo.encode()).hexdigest()[:5]}")
self.branchNamePrefix = branchNamePrefix
def progress(self) -> str:
@@ -28,12 +30,41 @@ class GitClone(AbstractStep[str]):
if not result or any(fetch_info.flags & fetch_info.ERROR for fetch_info in result):
raise BaseException(f"Failed to fetch updates for {self.env.repo}")
else:
git.Repo.clone_from(self.env.repo, self.temp_dir)
repo = git.Repo.init(self.temp_dir)
repo.remotes.append(repo.create_remote(
'origin',
self.env.repo
))
if self.branchNamePrefix != "":
repo.config_writer().set_value('remote "origin"',"fetch", f"+refs/heads/{self.branchNamePrefix}/*:refs/remotes/origin/{self.branchNamePrefix}/*").release()
repo.remotes.origin.fetch(prune=True)
if not os.path.exists(os.path.join(self.temp_dir, ".git")):
raise BaseException(f"Failed to clone repository {self.env.repo} to {self.temp_dir}")
return self.temp_dir
def get_branches(self) -> Dict[str, List[Any]]:
repo = git.Repo(self.temp_dir)
result = {}
for ref in repo.refs:
if ref.name.startswith('origin/') and not ref.name.startswith('origin/HEAD'):
branch_name = ref.name [len('origin/'):]
if not branch_name.startswith('auto/'): # Skip auto branches
result[branch_name] = []
cmt = [ref.commit]
for _ in range(10):
for commit in cmt:
result[branch_name].append({
'hexsha': commit.hexsha,
'author': commit.author.name,
'date': commit.committed_datetime.isoformat(),
'message': commit.message.strip()
})
cmt = [p for c in cmt for p in c.parents]
return result
import hashlib
@dataclass
+5 -1
View File
@@ -8,12 +8,13 @@ services:
- "5001:5001"
volumes:
- /var/run/docker.sock:/var/run/docker.sock
- tmp:/tmp
- /etc/passwd:/etc/passwd
environment:
- PYTHONUNBUFFERED=1
- VIRTUAL_HOST=brencher.rudolf.keenetic.link
- LETSENCRYPT_HOST=brencher.rudolf.keenetic.link
- PROFILES=brencher
- PROFILES=brencher,torrserv_proxy
deploy:
replicas: 1
restart_policy:
@@ -29,6 +30,9 @@ secrets:
brencher-secrets:
external: true
volumes:
tmp:
networks:
brencher-net:
driver: overlay
+20 -5
View File
@@ -231,11 +231,26 @@ function renderJobs() {
jobsList.innerHTML = environmentsRaw.map(([envObj, jobsArr]) => {
return `
<div class="env-jobs">
<h4>Environment: ${envObj.name || envObj.id || ''}</h4>
${Array.isArray(jobsArr) && jobsArr.length > 0
? jobsArr.map(job => `
<div class="job-item"><strong>${job.env} - ${job.name}</strong> - ${JSON.stringify(job.status)}</div>`).join('')
: '<div class="job-item">No jobs found.</div>'}
<h4>Environment: ${envObj.name || envObj.id || ''}</h4>
${Array.isArray(jobsArr) && jobsArr.length > 0
? jobsArr.map(job => {
let statusDisplay = '';
let tooltip = '';
if (Array.isArray(job.status) && job.status.length > 0) {
statusDisplay = typeof job.status[0] === 'string' ? job.status[0] : JSON.stringify(job.status[0]);
if (job.status.length > 1) {
tooltip = job.status.slice(1).flatMap(s => typeof s === 'string' ? [s] : s).join('<br/>');
}
} else {
statusDisplay = typeof job.status === 'string' ? job.status : JSON.stringify(job.status);
}
return `
<div class="job-item">
<strong>${job.env} - ${job.name}</strong> - ${statusDisplay}
${tooltip ? `<span class="tooltip-icon" title="${tooltip.replace(/"/g, '&quot;')}">&#x1F6C8;</span>` : ''}
</div>`;
}).join('')
: '<div class="job-item">No jobs found.</div>'}
</div>`;
}).join('');
}