before deploy to box
This commit is contained in:
@@ -1,3 +1,11 @@
|
||||
docker run -it --rm \
|
||||
-v /media/DATA/src/brencher:/br \
|
||||
-v /var/run/docker.sock:/var/run/docker.sock \
|
||||
registry.rudolf.keenetic.link/brencher:auto-b62e6 \
|
||||
/bin/bash
|
||||
|
||||
|
||||
|
||||
# Startup config .env
|
||||
- Git repository url
|
||||
- Authentication username
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
Use language python.
|
||||
|
||||
|
||||
Locally run in venv create with uv tool.
|
||||
Use venv with uv for Python environment management and dependency installation.
|
||||
Keep requirements.txt updated.
|
||||
|
||||
@@ -1,13 +0,0 @@
|
||||
ktor {
|
||||
deployment {
|
||||
port = 8080
|
||||
}
|
||||
application {
|
||||
modules = [ org.rudolf.ApplicationKt.module ]
|
||||
}
|
||||
include required(classpath("local.conf"))
|
||||
git {
|
||||
repositoryUrl = "https://github.com/rudolf1/brencher.git"
|
||||
branchRefreshIntervalMinutes = 5
|
||||
}
|
||||
}
|
||||
@@ -21,6 +21,8 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
# Load environment variables from .env file
|
||||
load_dotenv("local.env")
|
||||
load_dotenv('/run/secrets/brencher-secrets')
|
||||
|
||||
app = Flask(__name__, static_folder='frontend')
|
||||
socketio = SocketIO(app, cors_allowed_origins="*")
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ from steps.step import AbstractStep
|
||||
|
||||
env = Environment(
|
||||
id="brencher",
|
||||
branches=["main"],
|
||||
branches=["main", "small_fix"],
|
||||
state="Active",
|
||||
repo="https://github.com/rudolf1/brencher.git",
|
||||
)
|
||||
@@ -16,19 +16,29 @@ env = Environment(
|
||||
|
||||
def create_pipeline(env: Environment) -> List[AbstractStep]:
|
||||
clone = GitClone(env)
|
||||
checkoutMerged = CheckoutMerged(clone, env.branches, env=env)
|
||||
checkoutMerged = CheckoutMerged(clone, env.branches, env=env,
|
||||
push = False,
|
||||
git_user_email="rudolfss13@gmail.com",
|
||||
git_user_name="brencher_bot"
|
||||
)
|
||||
|
||||
# git config --global user.email "rudolfss13@gmail.com"
|
||||
# git config --global user.name "brencher_bot"
|
||||
|
||||
|
||||
buildDocker = DockerComposeBuild(clone,
|
||||
docker_repo_username = "",
|
||||
docker_repo_password = "",
|
||||
docker_compose_path = "docker-compose.yml",
|
||||
docker_repo_url="https://registry.rudolf.keenetic.link",
|
||||
publish=False,
|
||||
envs = lambda: { "version": "auto-" + checkoutMerged.result[1][0:5] },
|
||||
envs = lambda: { "version": "auto-" + checkoutMerged.result.version },
|
||||
env=env
|
||||
)
|
||||
deployDocker = DockerSwarmDeploy(
|
||||
wd=clone,
|
||||
envs = lambda: { "version": "auto-" + checkoutMerged.result[1][0:5] },
|
||||
buildDocker=buildDocker,
|
||||
envs = lambda: { "version": "auto-" + checkoutMerged.result.version },
|
||||
stack_name = "brencher",
|
||||
docker_compose_path = "docker-compose.yml",
|
||||
env=env,
|
||||
@@ -37,7 +47,12 @@ def create_pipeline(env: Environment) -> List[AbstractStep]:
|
||||
clone,
|
||||
checkoutMerged,
|
||||
buildDocker,
|
||||
deployDocker
|
||||
deployDocker,
|
||||
# TODO add health check.
|
||||
# TODO add reverse flow. We need to understand what is deployed
|
||||
# TODO add teamcity support
|
||||
# TODO add udploy support
|
||||
|
||||
]
|
||||
|
||||
brencher: Tuple[Environment, List[AbstractStep]] = (env, create_pipeline(env))
|
||||
|
||||
+82
-36
@@ -37,19 +37,20 @@ class DockerComposeBuild(AbstractStep[List[str]]):
|
||||
images_built = []
|
||||
try:
|
||||
# Authenticate to docker repo
|
||||
client = docker.from_env()
|
||||
client = docker.DockerClient(base_url='unix://var/run/docker.sock')
|
||||
if self.publish:
|
||||
client.login(username=self.docker_repo_username, password=self.docker_repo_password, registry=self.docker_repo_url)
|
||||
env = self.envs()
|
||||
# Parse docker-compose file
|
||||
with open(self.docker_compose_path, 'r') as f:
|
||||
docker_compose_absolute_path = os.path.join(self.wd.result, self.docker_compose_path)
|
||||
with open(docker_compose_absolute_path, 'r') as f:
|
||||
content = f.read()
|
||||
content = re.sub(r'\$\{([^}]+)\}', lambda m: env[m.group(1)], content)
|
||||
compose = yaml.safe_load(content)
|
||||
services = compose.get('services', {})
|
||||
|
||||
for name, svc in services.items():
|
||||
build_ctx = svc.get('build')
|
||||
build_ctx = os.path.join(self.wd.result, svc.get('build'))
|
||||
image = svc.get('image')
|
||||
if not build_ctx or not image:
|
||||
continue
|
||||
@@ -82,54 +83,99 @@ class DockerComposeBuild(AbstractStep[List[str]]):
|
||||
except Exception as e:
|
||||
raise e
|
||||
|
||||
class DockerSwarmDeploy(AbstractStep):
|
||||
class DockerSwarmDeploy(AbstractStep[str]):
|
||||
def __init__(self,
|
||||
wd: GitClone,
|
||||
buildDocker: DockerComposeBuild,
|
||||
docker_compose_path: str,
|
||||
envs: Callable[[], Dict[str, str]],
|
||||
stack_name: str,
|
||||
**kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self.wd = wd
|
||||
self.buildDocker = buildDocker
|
||||
self.envs = envs
|
||||
self.docker_compose_path = docker_compose_path
|
||||
self.stack_name = stack_name
|
||||
|
||||
def progress(self) -> None:
|
||||
def _get_stack_state(self, stack_name):
|
||||
current_services = {}
|
||||
cmd_ps = [
|
||||
"docker", "service", "ls", "--format", "{{.Name}}"
|
||||
]
|
||||
ps_result = subprocess.run(cmd_ps, capture_output=True, text=True, check=True)
|
||||
stack_svc_names = ps_result.stdout.strip().splitlines()
|
||||
for svc_name in stack_svc_names:
|
||||
if not svc_name.startswith(stack_name + "_"):
|
||||
continue
|
||||
cmd_inspect = [
|
||||
"docker", "service", "inspect", svc_name, "--format", "{{.Spec.TaskTemplate.ContainerSpec.Image}}"
|
||||
]
|
||||
inspect_result = subprocess.run(cmd_inspect, capture_output=True, text=True, check=True)
|
||||
image_tag = inspect_result.stdout.strip()
|
||||
service_name = svc_name.removeprefix(self.stack_name + "_")
|
||||
current_services[service_name] = image_tag
|
||||
logger.info(f"Current services {current_services}")
|
||||
return current_services
|
||||
|
||||
def progress(self) -> str:
|
||||
"""
|
||||
Deploys to Docker Swarm using the specified docker-compose.yaml.
|
||||
"""
|
||||
if isinstance(self.wd.result, BaseException):
|
||||
raise self.wd.result
|
||||
try:
|
||||
env = self.envs()
|
||||
# Prepare docker-compose file with env substitution
|
||||
docker_compose_absolute_path = os.path.join(self.wd.result, self.docker_compose_path)
|
||||
with open(docker_compose_absolute_path, 'r') as f:
|
||||
content = f.read()
|
||||
content = re.sub(r'\$\{([^}]+)\}', lambda m: env.get(m.group(1), ""), content)
|
||||
compose = yaml.safe_load(content)
|
||||
if "services" in compose:
|
||||
for svc in compose["services"].values():
|
||||
if "build" in svc:
|
||||
del svc["build"]
|
||||
content = yaml.safe_dump(compose)
|
||||
tmp_compose_path = docker_compose_absolute_path + ".tmp"
|
||||
with open(tmp_compose_path, 'w') as f:
|
||||
f.write(content)
|
||||
if isinstance(self.buildDocker.result, BaseException):
|
||||
raise self.buildDocker.result
|
||||
|
||||
# Deploy stack to Docker Swarm
|
||||
cmd = [
|
||||
"docker", "stack", "deploy",
|
||||
"-c", tmp_compose_path,
|
||||
self.stack_name
|
||||
]
|
||||
logger.info(f"Deploying stack '{self.stack_name}' using {tmp_compose_path}")
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if result.returncode != 0:
|
||||
logger.error(f"Stack deploy failed: {result.stderr}")
|
||||
raise RuntimeError(f"Stack deploy failed: {result.stderr}")
|
||||
logger.info(f"Stack deployed successfully: {result.stdout}")
|
||||
except Exception as e:
|
||||
logger.error(f"DockerSwarmDeploy failed: {e}")
|
||||
raise
|
||||
# TODO skip if correct stack already deployed
|
||||
# Only image tags compared. Need more.
|
||||
# TODO I guess label for container should be added.
|
||||
# TODO Or find keys for docker swarm to skip deploy if no changes.
|
||||
|
||||
env = self.envs()
|
||||
# Prepare docker-compose file with env substitution
|
||||
docker_compose_absolute_path = os.path.join(self.wd.result, self.docker_compose_path)
|
||||
with open(docker_compose_absolute_path, 'r') as f:
|
||||
content = f.read()
|
||||
content = re.sub(r'\$\{([^}]+)\}', lambda m: env.get(m.group(1), ""), content)
|
||||
compose = yaml.safe_load(content)
|
||||
if "services" in compose:
|
||||
for svc in compose["services"].values():
|
||||
if "build" in svc:
|
||||
del svc["build"]
|
||||
content = yaml.safe_dump(compose)
|
||||
|
||||
current_services = self._get_stack_state(self.stack_name)
|
||||
expected_services = compose.get("services", {})
|
||||
diffs = []
|
||||
ok = []
|
||||
for svc_name, svc in expected_services.items():
|
||||
expected_image = svc.get("image")
|
||||
running_image = current_services.get(svc_name)
|
||||
l = f"Service '{svc_name}' (expected image: '{expected_image}', actual image: '{running_image}')"
|
||||
logger.info(l)
|
||||
if running_image and expected_image and running_image == expected_image:
|
||||
ok.append(l)
|
||||
else:
|
||||
diffs.append(l)
|
||||
|
||||
if len(diffs) == 0:
|
||||
return '\\n'.join(ok)
|
||||
logger.info(f"Diff {diffs}")
|
||||
tmp_compose_path = docker_compose_absolute_path + ".tmp"
|
||||
with open(tmp_compose_path, 'w') as f:
|
||||
f.write(content)
|
||||
|
||||
# Deploy stack to Docker Swarm
|
||||
cmd = [
|
||||
"docker", "stack", "deploy",
|
||||
"-c", tmp_compose_path,
|
||||
self.stack_name
|
||||
]
|
||||
logger.info(f"Deploying stack '{self.stack_name}' using {tmp_compose_path}")
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if result.returncode != 0:
|
||||
logger.error(f"Stack deploy failed: {result.stderr}")
|
||||
raise RuntimeError(f"Stack deploy failed: {result.stderr}")
|
||||
logger.info(f"Stack deployed successfully: {result.stdout}")
|
||||
return f"Stack deployed successfully: {result.stdout}"
|
||||
|
||||
+61
-41
@@ -35,14 +35,27 @@ class GitClone(AbstractStep[str]):
|
||||
|
||||
import hashlib
|
||||
|
||||
class CheckoutMerged(AbstractStep[Tuple[str, str]]):
|
||||
@dataclass
|
||||
class CheckoutAndMergeResult:
|
||||
branch_name: str
|
||||
commit_hash: str
|
||||
version: str
|
||||
|
||||
class CheckoutMerged(AbstractStep[CheckoutAndMergeResult]):
|
||||
wd: GitClone
|
||||
branches: List[str]
|
||||
|
||||
def __init__(self, wd: GitClone, branches: List[str], **kwargs):
|
||||
def __init__(self, wd: GitClone, branches: List[str],
|
||||
git_user_email,
|
||||
git_user_name,
|
||||
push: bool = True,
|
||||
**kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self.wd = wd
|
||||
self.branches = branches
|
||||
self.git_user_email = git_user_email
|
||||
self.git_user_name = git_user_name
|
||||
self.push = push
|
||||
|
||||
def _commits_tree(self, repo: git.Repo) -> Dict[Commit, List[Commit]]:
|
||||
childs: Dict[Commit, List[Commit]] = {}
|
||||
@@ -66,7 +79,7 @@ class CheckoutMerged(AbstractStep[Tuple[str, str]]):
|
||||
|
||||
return visited
|
||||
|
||||
def progress(self) -> Tuple[str, str]:
|
||||
def progress(self) -> CheckoutAndMergeResult:
|
||||
if self.wd.result_obj is BaseException:
|
||||
raise self.wd.result_obj
|
||||
|
||||
@@ -79,22 +92,28 @@ class CheckoutMerged(AbstractStep[Tuple[str, str]]):
|
||||
|
||||
# Open the repository
|
||||
repo = git.Repo(repo_path)
|
||||
|
||||
# Set user email and name for the repo
|
||||
with repo.config_writer() as cw:
|
||||
cw.set_value("user", "email", self.git_user_email)
|
||||
cw.set_value("user", "name", self.git_user_name)
|
||||
logger.info(f"Selected branches: {self.branches}")
|
||||
# Extract commit ids for the selected branches
|
||||
commit_ids: Dict[Commit, str] = {}
|
||||
for branch in self.branches:
|
||||
commit = repo.commit(f'origin/{branch}')
|
||||
commit_ids[commit] = branch
|
||||
for br in self.branches:
|
||||
commit = repo.commit(f'origin/{br}')
|
||||
commit_ids[commit] = br
|
||||
|
||||
logger.info(f"Commit ids for branches: {commit_ids}")
|
||||
|
||||
tree = self._commits_tree(repo)
|
||||
|
||||
merge_commit = [(c, self._find_merge_childs(tree, c)) for c in commit_ids.keys()]
|
||||
result = merge_commit[0]
|
||||
result = set(merge_commit[0][1])
|
||||
for c, l in merge_commit[1:]:
|
||||
inters = set(result).intersection(set(l))
|
||||
if len(inters) == 0:
|
||||
result = set(result).intersection(set(l))
|
||||
if len(result) == 0:
|
||||
break
|
||||
|
||||
if len(result) > 0:
|
||||
merge_commit = [c for c in merge_commit[0][1] if c in result][0]
|
||||
logger.info(f"Common commit found {merge_commit}")
|
||||
@@ -102,24 +121,30 @@ class CheckoutMerged(AbstractStep[Tuple[str, str]]):
|
||||
merge_commit = None
|
||||
logger.info(f"Common commit not found")
|
||||
|
||||
if merge_commit is not None:
|
||||
for head in repo.branches:
|
||||
if head.commit.hexsha == merge_commit.hexsha:
|
||||
logger.info(f"Merge commit {merge_commit} corresponds to branch {head}")
|
||||
return (head.name, merge_commit.hexsha)
|
||||
sorted_commits = sorted(commit_ids.keys(), key=lambda x: x.hexsha)
|
||||
auto_branch_hash = hashlib.sha1(''.join([x.hexsha for x in sorted_commits]).encode()).hexdigest()
|
||||
logger.info(f"Commit ids for branches: {commit_ids}")
|
||||
|
||||
auto_branch_name = f"auto/{auto_branch_hash}"
|
||||
version = '-'.join([x.hexsha[0:5] for x in sorted_commits])
|
||||
if merge_commit is not None:
|
||||
for head in repo.branches:
|
||||
if head.is_remote and head.commit.hexsha == merge_commit.hexsha:
|
||||
logger.info(f"Merge commit {merge_commit} corresponds to branch {head}")
|
||||
return CheckoutAndMergeResult(head.name, merge_commit.hexsha, version)
|
||||
# TODO No sense to generate branch name from hash. Just use commit it after merge.
|
||||
# TODO Not sure. Think about it.
|
||||
# TODO Write test to cover usecases.
|
||||
# TODO Blocker!!! Not able to push in docker.
|
||||
# TODO Blocker!!! Do not lookup local branches.
|
||||
|
||||
|
||||
if merge_commit is not None:
|
||||
repo.git.branch('-f', auto_branch_name, merge_commit.hexsha)
|
||||
repo.git.checkout(auto_branch_name)
|
||||
repo.git.push('-f', 'origin', auto_branch_name)
|
||||
logger.info(f"Head is {repo.head.commit}")
|
||||
logger.info(f"Pushed {merge_commit} to {auto_branch_name}")
|
||||
return (auto_branch_name, merge_commit.hexsha)
|
||||
if self.push:
|
||||
repo.git.push('-f', 'origin', auto_branch_name)
|
||||
logger.info(f"Pushed {merge_commit} to {auto_branch_name}")
|
||||
return CheckoutAndMergeResult(auto_branch_name, merge_commit.hexsha, version)
|
||||
|
||||
logger.info(f"Looking for existing auto branch: {auto_branch_name}")
|
||||
|
||||
@@ -131,40 +156,35 @@ class CheckoutMerged(AbstractStep[Tuple[str, str]]):
|
||||
repo.git.checkout(auto_branch_name)
|
||||
auto_branch_hash = repo.head.commit.hexsha
|
||||
|
||||
return (auto_branch_name, auto_branch_hash)
|
||||
return CheckoutAndMergeResult(auto_branch_name, auto_branch_hash, version)
|
||||
|
||||
repo.git.checkout(repo.head.commit.hexsha)
|
||||
if auto_branch_name in repo.branches:
|
||||
repo.git.branch('-D', auto_branch_name)
|
||||
# Auto branch doesn't exist, create it
|
||||
logger.info(f"Creating new auto branch: {auto_branch_name}")
|
||||
|
||||
# Start with the first branch
|
||||
base_branch = self.branches[0]
|
||||
repo.git.checkout(base_branch)
|
||||
|
||||
# Create temporary branch for merging
|
||||
temp_branch = f"temp-merge-{auto_branch_hash}"
|
||||
# Remove temp_branch if it exists
|
||||
if temp_branch in repo.heads:
|
||||
repo.git.branch('-D', temp_branch)
|
||||
repo.git.checkout('-b', temp_branch)
|
||||
commits = list(commit_ids.keys())
|
||||
repo.git.branch('-f', auto_branch_name, commits[0].hexsha)
|
||||
repo.git.checkout(auto_branch_name)
|
||||
|
||||
# Merge the rest of the branches
|
||||
for branch in self.branches[1:]:
|
||||
for commit in commits[1:]:
|
||||
try:
|
||||
logger.info(f"Merging branch: {branch}")
|
||||
repo.git.merge(branch, '--no-ff')
|
||||
logger.info(f"Merging commit: {commit}")
|
||||
repo.git.merge(commit, '--no-ff')
|
||||
except git.GitCommandError as e:
|
||||
# Handle merge conflicts according to predefined rules
|
||||
# For now, we'll abort the merge and report failure
|
||||
repo.git.merge('--abort')
|
||||
error_message = f"Merge conflict when merging {branch}: {str(e)}"
|
||||
error_message = f"Merge conflict when merging {commit}: {str(e)}"
|
||||
logger.error(error_message)
|
||||
|
||||
raise BaseException(error_message)
|
||||
|
||||
# Create and push the auto branch
|
||||
repo.git.branch('-f', auto_branch_name)
|
||||
repo.git.checkout(auto_branch_name)
|
||||
auto_branch_hash = repo.head.commit.hexsha
|
||||
repo.git.push('-f', 'origin', auto_branch_name)
|
||||
|
||||
return (auto_branch_name,auto_branch_hash)
|
||||
auto_branch_hash = repo.head.commit.hexsha
|
||||
if self.push:
|
||||
logger.info(f"Pushing {auto_branch_hash} to {auto_branch_name}")
|
||||
repo.git.push('-f', 'origin', auto_branch_name)
|
||||
return CheckoutAndMergeResult(auto_branch_name,auto_branch_hash, version)
|
||||
|
||||
@@ -3,10 +3,13 @@ services:
|
||||
brencher-backend:
|
||||
build: .
|
||||
image: registry.rudolf.keenetic.link/brencher:${version}
|
||||
user: 1000:998
|
||||
ports:
|
||||
- "5000:5000"
|
||||
container_name: brencher-backend
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
- /etc/passwd:/etc/passwd
|
||||
environment:
|
||||
- PYTHONUNBUFFERED=1
|
||||
deploy:
|
||||
@@ -15,6 +18,12 @@ services:
|
||||
condition: on-failure
|
||||
networks:
|
||||
- brencher-net
|
||||
secrets:
|
||||
- brencher-secrets
|
||||
|
||||
secrets:
|
||||
brencher-secrets:
|
||||
external: true
|
||||
|
||||
networks:
|
||||
brencher-net:
|
||||
|
||||
Reference in New Issue
Block a user