Copilot/replace existing typecheckers with py changes (#32)
* ty and pylance fixed. Flake left
This commit is contained in:
@@ -0,0 +1,9 @@
|
||||
[flake8]
|
||||
exclude =
|
||||
.venv,
|
||||
__pycache__,
|
||||
.git,
|
||||
.pytest_cache,
|
||||
*.egg-info
|
||||
max-line-length = 127
|
||||
max-complexity = 10
|
||||
@@ -17,25 +17,24 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
python-version: ["3.9", "3.10", "3.11"]
|
||||
python-version: ["3.11"]
|
||||
checker: [
|
||||
"flake8 . --count --select=E9,F63,F7,F82 --ignore=F824 --show-source --statistics",
|
||||
"flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics",
|
||||
# "ty check backend",
|
||||
"mypy backend --config-file mypy.ini",
|
||||
"pytest",
|
||||
]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
- name: Set up Python ${{ matrix.python-version }}, ${{ matrix.checker }}"
|
||||
uses: actions/setup-python@v3
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
python -m pip install flake8 pytest
|
||||
if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
|
||||
- name: Lint with flake8
|
||||
run: |
|
||||
# stop the build if there are Python syntax errors or undefined names
|
||||
flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
|
||||
# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
|
||||
flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
|
||||
- name: Test with pytest
|
||||
run: |
|
||||
pytest
|
||||
python -m pip install flake8 pytest ty
|
||||
pip install -r requirements.txt
|
||||
${{ matrix.checker }}
|
||||
|
||||
Vendored
+4
@@ -1,6 +1,10 @@
|
||||
{
|
||||
"recommendations": [
|
||||
"ms-python.vscode-pylance",
|
||||
"vinirossa.vscode-gitandgithub-pack",
|
||||
"astral-sh.ty"
|
||||
"ms-python.mypy-type-checker",
|
||||
"ms-python.python",
|
||||
"vinirossa.vscode-gitandgithub-pack"
|
||||
]
|
||||
}
|
||||
Vendored
+18
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"ty.enable": true
|
||||
"python.linting.enabled": true,
|
||||
"python.linting.mypyEnabled": true,
|
||||
"python.linting.mypyArgs": [
|
||||
"--config-file=mypy.ini"
|
||||
],
|
||||
"python.analysis.typeCheckingMode": "basic",
|
||||
"python.analysis.diagnosticMode": "workspace",
|
||||
"[python]": {
|
||||
"editor.formatOnSave": false,
|
||||
"editor.codeActionsOnSave": {
|
||||
"source.organizeImports": "explicit"
|
||||
}
|
||||
},
|
||||
"mypy.configFile": "mypy.ini",
|
||||
"mypy.runUsingActiveInterpreter": true
|
||||
}
|
||||
@@ -19,10 +19,12 @@ Generate docker compose file to deploy to docker swarm.
|
||||
|
||||
python ./backend/app.py brencher_local1 noweb
|
||||
|
||||
# Local start
|
||||
|
||||
python ./backend/app.py brencher_local1 noweb
|
||||
|
||||
# Update docker secret
|
||||
|
||||
|
||||
|
||||
docker service scale brencher_brencher-backend=0 brencher2_brencher-backend=0
|
||||
docker service update --secret-rm brencher-secrets brencher_brencher-backend
|
||||
docker service update --secret-rm brencher-secrets brencher2_brencher-backend
|
||||
|
||||
+48
-53
@@ -1,25 +1,21 @@
|
||||
from multiprocessing import Pipe
|
||||
import os
|
||||
import json
|
||||
import time
|
||||
import threading
|
||||
import hashlib
|
||||
from flask import Flask, request, jsonify, send_from_directory
|
||||
from flask import Flask, send_from_directory
|
||||
from steps.step import AbstractStep
|
||||
from flask_socketio import SocketIO, emit
|
||||
import socketio as socketio_client
|
||||
from dotenv import load_dotenv
|
||||
import git
|
||||
import enironment
|
||||
from steps.git import GitClone
|
||||
import tempfile
|
||||
import logging
|
||||
from dataclasses import dataclass, asdict, field
|
||||
from dataclasses import asdict
|
||||
from typing import List, Dict, Any, Optional, Tuple
|
||||
import shutil
|
||||
import subprocess
|
||||
import traceback
|
||||
from typing import TypeVar
|
||||
import types
|
||||
from flask.json.provider import DefaultJSONProvider
|
||||
|
||||
# Configure logging
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
||||
@@ -30,8 +26,9 @@ logger = logging.getLogger(__name__)
|
||||
load_dotenv("local.env")
|
||||
load_dotenv('/run/secrets/brencher-secrets')
|
||||
|
||||
|
||||
class DataclassJSONEncoder(json.JSONEncoder):
|
||||
def default(self, o):
|
||||
def default(self, o: Any) -> Any:
|
||||
if hasattr(o, '__dataclass_fields__'):
|
||||
return asdict(o)
|
||||
if isinstance(o, BaseException):
|
||||
@@ -41,17 +38,16 @@ class DataclassJSONEncoder(json.JSONEncoder):
|
||||
except TypeError:
|
||||
return str(o)
|
||||
|
||||
import types
|
||||
|
||||
custom_json = types.SimpleNamespace()
|
||||
custom_json.dumps = lambda obj, **kwargs: json.dumps(obj, cls=DataclassJSONEncoder, **kwargs)
|
||||
custom_json.loads = json.loads
|
||||
|
||||
from flask.json.provider import DefaultJSONProvider
|
||||
|
||||
class DataclassJSONProvider(DefaultJSONProvider):
|
||||
def dumps(self, obj, **kwargs):
|
||||
def dumps(self, obj: Any, **kwargs: Any) -> str:
|
||||
return json.dumps(obj, cls=DataclassJSONEncoder, **kwargs)
|
||||
def loads(self, s, **kwargs):
|
||||
def loads(self, s: str | bytes, **kwargs: Any) -> Any:
|
||||
return json.loads(s, **kwargs)
|
||||
|
||||
app = Flask(__name__, static_folder='frontend')
|
||||
@@ -63,20 +59,19 @@ FRONTEND_DIR = os.path.join(os.path.dirname(__file__), '../frontend')
|
||||
# In-memory state
|
||||
environments: Dict[str, Tuple[enironment.Environment, List[AbstractStep]]] = {}
|
||||
environments_slaves: Dict[str, Tuple[enironment.Environment, List[AbstractStep]]] = {}
|
||||
branches = {}
|
||||
branches_slaves = {}
|
||||
branches: Dict[str, Dict[str, Any]] = {}
|
||||
branches_slaves: Dict[str, Dict[str, Any]] = {}
|
||||
state_lock = threading.Lock()
|
||||
|
||||
@app.route('/')
|
||||
def serve_index():
|
||||
def serve_index() -> Any:
|
||||
return send_from_directory(FRONTEND_DIR, 'index.html')
|
||||
|
||||
@app.route('/<path:path>')
|
||||
def serve_static(path):
|
||||
def serve_static(path: str) -> Any:
|
||||
return send_from_directory(FRONTEND_DIR, path)
|
||||
|
||||
# --- WebSocket Endpoints ---
|
||||
from flask import copy_current_request_context
|
||||
from flask_socketio import Namespace
|
||||
|
||||
T = TypeVar('T')
|
||||
@@ -84,26 +79,25 @@ def merge_dicts(a: Dict[str, T], b: Dict[str, T]) -> Dict[str, T]:
|
||||
result: Dict[str, T] = {}
|
||||
for k in (a.keys() | b.keys()):
|
||||
if k not in a or k not in b:
|
||||
result[k] = a.get(k, b.get(k)) # type: ignore[arg-type]
|
||||
result[k] = a.get(k, b.get(k)) # type: ignore[assignment]
|
||||
continue
|
||||
if (
|
||||
isinstance(a[k], dict)
|
||||
and isinstance(b[k], dict)
|
||||
):
|
||||
result[k] = merge_dicts(a[k], b[k]) # type: ignore[arg-type]
|
||||
result[k] = merge_dicts(a[k], b[k]) # type: ignore[arg-type,assignment]
|
||||
else:
|
||||
result[k] = b[k]
|
||||
return result
|
||||
|
||||
slave_url = os.getenv('SLAVE_BRENCHER')
|
||||
remote_sio: socketio_client.Client | None = None
|
||||
if slave_url:
|
||||
logger.info(f"SLAVE_BRENCHER set, will connect to master at {slave_url}")
|
||||
remote_sio = socketio_client.Client(logger=False, engineio_logger=False)
|
||||
|
||||
# Handlers for events from master -> re-emit locally (marked so we don't loop)
|
||||
@remote_sio.on('branches', namespace='/ws/branches')
|
||||
def _remote_branches(data):
|
||||
def _remote_branches(data: Any) -> None:
|
||||
global branches, branches_slaves
|
||||
branches_slaves = data
|
||||
merge_result = merge_dicts(branches, branches_slaves)
|
||||
@@ -113,7 +107,7 @@ if slave_url:
|
||||
logger.error(f"Error forwarding remote branches locally: {e}")
|
||||
|
||||
@remote_sio.on('environments', namespace='/ws/environment')
|
||||
def _remote_environments(data):
|
||||
def _remote_environments(data: Any) -> None:
|
||||
global environments, environments_slaves
|
||||
environments_slaves = data
|
||||
try:
|
||||
@@ -122,32 +116,32 @@ if slave_url:
|
||||
logger.error(f"Error forwarding remote environments locally: {e}")
|
||||
|
||||
@remote_sio.on('error', namespace='/ws/errors')
|
||||
def _remote_error(data):
|
||||
def _remote_error(data: Any) -> None:
|
||||
try:
|
||||
socketio.emit('error', data, namespace='/ws/errors')
|
||||
except Exception as e:
|
||||
logger.error(f"Error forwarding remote errors locally: {e}")
|
||||
|
||||
@remote_sio.event
|
||||
def connect():
|
||||
def connect() -> None:
|
||||
logger.info("Connected to master brencher (SLAVE mode).")
|
||||
|
||||
@remote_sio.event
|
||||
def disconnect():
|
||||
def disconnect() -> None:
|
||||
logger.info("Disconnected from master brencher (SLAVE mode).")
|
||||
|
||||
|
||||
class BranchesNamespace(Namespace):
|
||||
def on_connect(self, auth=None):
|
||||
def on_connect(self, auth: Optional[Any] = None) -> None:
|
||||
emit('branches', merge_dicts(branches, branches_slaves))
|
||||
def on_update(self, data):
|
||||
def on_update(self, data: Any) -> None:
|
||||
emit('branches', merge_dicts(branches, branches_slaves))
|
||||
|
||||
def get_local_envs_to_emit() -> Dict[str, Tuple[enironment.Environment, List[AbstractStep]]]:
|
||||
env_dtos = {}
|
||||
def get_local_envs_to_emit() -> Dict[str, Tuple[Dict[str, Any], List[Dict[str, Any]]]]:
|
||||
env_dtos: Dict[str, Tuple[Dict[str, Any], List[Dict[str, Any]]]] = {}
|
||||
for e, p in environments.values():
|
||||
env = asdict(e)
|
||||
res = []
|
||||
res: List[Dict[str, Any]] = []
|
||||
for r in p:
|
||||
if isinstance(r.result_obj, BaseException):
|
||||
stack = traceback.format_exception(type(r.result_obj), r.result_obj, r.result_obj.__traceback__)
|
||||
@@ -163,12 +157,12 @@ def get_local_envs_to_emit() -> Dict[str, Tuple[enironment.Environment, List[Abs
|
||||
env_dtos[env['id']] = (env, res)
|
||||
return env_dtos
|
||||
|
||||
def get_global_envs_to_emit():
|
||||
def get_global_envs_to_emit() -> Any:
|
||||
global environments, environments_slaves
|
||||
local_envs = get_local_envs_to_emit()
|
||||
merge_result = merge_dicts(local_envs, environments_slaves)
|
||||
logger.info(f"Local keys: {local_envs.keys()}")
|
||||
logger.info(f"Slave keys: {environments_slaves.keys()}")
|
||||
merge_result = merge_dicts(local_envs, environments_slaves) # type: ignore[misc]
|
||||
# logger.info(f"Local keys: {local_envs.keys()}")
|
||||
# logger.info(f"Slave keys: {environments_slaves.keys()}")
|
||||
common_keys = set(local_envs.keys()) & set(environments_slaves.keys())
|
||||
if len(common_keys) > 0:
|
||||
socketio.emit('error', {'message': f"Conflict: both master and slave have environment with id {common_keys}"}, namespace='/ws/errors')
|
||||
@@ -177,15 +171,15 @@ def get_global_envs_to_emit():
|
||||
environment_update_event = threading.Event()
|
||||
|
||||
@app.route('/state')
|
||||
def serve_state():
|
||||
def serve_state() -> Any:
|
||||
return get_global_envs_to_emit()
|
||||
|
||||
class EnvironmentNamespace(Namespace):
|
||||
|
||||
def on_connect(self, auth=None):
|
||||
def on_connect(self, auth: Optional[Any] = None) -> None:
|
||||
emit('environments', get_global_envs_to_emit())
|
||||
|
||||
def on_update(self, data):
|
||||
def on_update(self, data: Dict[str, Any]) -> None:
|
||||
global environments, remote_sio
|
||||
|
||||
logger.info(f"Received environment update: {data}")
|
||||
@@ -204,9 +198,9 @@ class EnvironmentNamespace(Namespace):
|
||||
emit('environments', get_global_envs_to_emit(), namespace='/ws/environment')
|
||||
|
||||
class ErrorsNamespace(Namespace):
|
||||
def on_connect(self, auth=None):
|
||||
def on_connect(self, auth: Optional[Any] = None) -> None:
|
||||
pass
|
||||
def on_error(self, data):
|
||||
def on_error(self, data: Any) -> None:
|
||||
emit('error', data)
|
||||
|
||||
socketio.on_namespace(BranchesNamespace('/ws/branches'))
|
||||
@@ -219,7 +213,7 @@ if __name__ == '__main__':
|
||||
import os
|
||||
import sys
|
||||
|
||||
def sigchld_handler(signum, frame):
|
||||
def sigchld_handler(signum, frame): # type: ignore[no-untyped-def]
|
||||
"""Reap zombie processes"""
|
||||
while True:
|
||||
try:
|
||||
@@ -249,20 +243,21 @@ if __name__ == '__main__':
|
||||
environments = {e[0].id: e for e in environments_l}
|
||||
|
||||
import sys
|
||||
cli_env_ids = sys.argv[1:]
|
||||
if len(cli_env_ids) == 0:
|
||||
cli_env_ids = os.getenv('PROFILES', '')
|
||||
cli_env_ids_list = sys.argv[1:]
|
||||
cli_env_ids_str: str
|
||||
if len(cli_env_ids_list) == 0:
|
||||
cli_env_ids_str = os.getenv('PROFILES', '')
|
||||
else:
|
||||
cli_env_ids = cli_env_ids[0]
|
||||
logger.info(f"cli_env_ids {cli_env_ids}")
|
||||
if len(cli_env_ids) > 0 and cli_env_ids[0] == '-':
|
||||
cli_env_ids = cli_env_ids[1:].split(',')
|
||||
cli_env_ids_str = cli_env_ids_list[0]
|
||||
logger.info(f"cli_env_ids {cli_env_ids_str}")
|
||||
if len(cli_env_ids_str) > 0 and cli_env_ids_str[0] == '-':
|
||||
cli_env_ids = cli_env_ids_str[1:].split(',')
|
||||
cli_env_ids = [x for x in cli_env_ids if len(x) > 0]
|
||||
logger.info(f"cli_env_ids (minus) {cli_env_ids}")
|
||||
if cli_env_ids and len(cli_env_ids) > 0:
|
||||
environments = { k: e for k, e in environments.items() if k not in cli_env_ids }
|
||||
else:
|
||||
cli_env_ids = cli_env_ids.split(',')
|
||||
cli_env_ids = cli_env_ids_str.split(',')
|
||||
cli_env_ids = [x for x in cli_env_ids if len(x) > 0]
|
||||
logger.info(f"cli_env_ids {cli_env_ids}")
|
||||
if cli_env_ids and len(cli_env_ids) > 0:
|
||||
@@ -275,7 +270,7 @@ if __name__ == '__main__':
|
||||
logger.info(f"Resulting profiles {environments.keys()}")
|
||||
|
||||
# Background thread to refresh branches every 5 minutes
|
||||
def emit_fresh_branches():
|
||||
def emit_fresh_branches() -> None:
|
||||
global branches, branches_slaves, environments
|
||||
for k, e in environments.items():
|
||||
env, pipe = e
|
||||
@@ -294,10 +289,10 @@ if __name__ == '__main__':
|
||||
socketio.emit('branches', merge_dicts(branches, branches_slaves), namespace='/ws/branches')
|
||||
|
||||
|
||||
def processing_thread():
|
||||
def processing_thread() -> None:
|
||||
while True:
|
||||
import processing
|
||||
def emit_envs():
|
||||
def emit_envs() -> None:
|
||||
try:
|
||||
socketio.emit('environments', get_global_envs_to_emit(), namespace='/ws/environment')
|
||||
except Exception as e:
|
||||
@@ -316,7 +311,7 @@ if __name__ == '__main__':
|
||||
|
||||
|
||||
# Connect in background so server startup isn't blocked
|
||||
def _connect_remote():
|
||||
def _connect_remote() -> None:
|
||||
global remote_sio
|
||||
print("Connecting to SLAVE_BRENCHER...")
|
||||
while True:
|
||||
|
||||
@@ -2,7 +2,7 @@ from steps.checks import SimpleLog, UrlCheck
|
||||
from steps.git import GitClone, CheckoutMerged, GitUnmerge
|
||||
from steps.docker import DockerComposeBuild, DockerSwarmCheck, DockerSwarmDeploy
|
||||
from enironment import Environment
|
||||
from typing import List, Dict, Any, Optional, Tuple
|
||||
from typing import List, Tuple
|
||||
from steps.step import AbstractStep
|
||||
from configs.brencher2 import checkPingF
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@ from steps.checks import SimpleLog, UrlCheck
|
||||
from steps.git import GitClone, CheckoutMerged, GitUnmerge
|
||||
from steps.docker import DockerComposeBuild, DockerSwarmCheck, DockerSwarmDeploy
|
||||
from enironment import Environment
|
||||
from typing import List, Dict, Any, Optional, Tuple
|
||||
from typing import List, Any, Tuple
|
||||
from steps.step import AbstractStep
|
||||
|
||||
env = Environment(
|
||||
@@ -13,7 +13,7 @@ env = Environment(
|
||||
repo="https://github.com/rudolf1/brencher.git",
|
||||
)
|
||||
|
||||
def checkPingF(obj: Any):
|
||||
def checkPingF(obj: Any) -> None:
|
||||
if not isinstance(obj, dict):
|
||||
raise TypeError(f"Expected dict, got {type(obj).__name__}")
|
||||
if "brencher" not in obj or "brencher2" not in obj:
|
||||
|
||||
@@ -2,7 +2,7 @@ from steps.git import GitClone, CheckoutMerged, GitUnmerge
|
||||
from steps.docker import DockerComposeBuild
|
||||
from steps.docker import DockerSwarmDeploy, DockerSwarmCheck
|
||||
from enironment import Environment
|
||||
from typing import List, Dict, Any, Optional, Tuple
|
||||
from typing import List, Tuple
|
||||
from steps.step import AbstractStep
|
||||
|
||||
env_local = Environment(
|
||||
|
||||
@@ -2,7 +2,7 @@ from steps.git import GitClone, CheckoutMerged, GitUnmerge
|
||||
from steps.docker import DockerComposeBuild
|
||||
from steps.docker import DockerSwarmDeploy, DockerSwarmCheck
|
||||
from enironment import Environment
|
||||
from typing import List, Dict, Any, Optional, Tuple
|
||||
from typing import List, Tuple
|
||||
from steps.step import AbstractStep
|
||||
|
||||
env_local = Environment(
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from steps.git import GitClone, CheckoutMerged, GitUnmerge
|
||||
from steps.docker import DockerComposeBuild, DockerSwarmCheck, DockerSwarmDeploy
|
||||
from steps.docker import DockerSwarmCheck, DockerSwarmDeploy
|
||||
from enironment import Environment
|
||||
from typing import List, Dict, Any, Optional, Tuple
|
||||
from typing import List, Tuple
|
||||
from steps.step import AbstractStep
|
||||
from steps.checks import SimpleLog, UrlCheck
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from steps.git import GitClone, CheckoutMerged, GitUnmerge
|
||||
from steps.docker import DockerComposeBuild, DockerSwarmCheck, DockerSwarmDeploy
|
||||
from enironment import Environment
|
||||
from typing import List, Dict, Any, Optional, Tuple
|
||||
from typing import List, Tuple
|
||||
from steps.step import AbstractStep
|
||||
|
||||
env = Environment(
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
from dataclasses import dataclass, asdict, field
|
||||
from typing import List, Dict, Any, Optional, Tuple, Callable
|
||||
from dataclasses import dataclass
|
||||
from typing import List, Tuple
|
||||
|
||||
@dataclass
|
||||
class Environment:
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from enironment import Environment
|
||||
from steps.step import AbstractStep
|
||||
from steps.git import GitUnmerge
|
||||
from typing import List, Dict, Any, Optional, Tuple, Callable
|
||||
from typing import List, Tuple, Callable
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -9,7 +9,7 @@ logger = logging.getLogger(__name__)
|
||||
def process_all_jobs(
|
||||
environemnts: List[Tuple[Environment, List[AbstractStep]]],
|
||||
onupdate: Callable[[], None]
|
||||
):
|
||||
) -> None:
|
||||
for env, pipe in environemnts:
|
||||
for step in pipe:
|
||||
try:
|
||||
@@ -19,7 +19,7 @@ def process_all_jobs(
|
||||
# TODO Move to separate job.
|
||||
# If branches list empty, need to find any brunch which includes commit and add pair (branch, commit)
|
||||
# If branches not empty, need to find most priority branch (project specific) and add (branch, HEAD)
|
||||
env.branches = [ (b1, c) for c,b in step.result for b1 in b ]
|
||||
env.branches= step.result
|
||||
logger.error(f"Branches on startup resolved {env.id}, job {step.name}: {env.branches}")
|
||||
except BaseException as e:
|
||||
error_msg = f"Error processing release {env.id}, job {step.name}: {str(e)}"
|
||||
|
||||
+4
-10
@@ -1,20 +1,14 @@
|
||||
import tempfile
|
||||
import git
|
||||
from git.objects import Commit
|
||||
from git.refs import Reference
|
||||
import logging
|
||||
from dataclasses import dataclass, asdict, field
|
||||
from typing import List, Optional, Union, Tuple, Set, Iterator, Dict, Any
|
||||
from typing import Any
|
||||
from steps.step import AbstractStep
|
||||
import tempfile
|
||||
import os
|
||||
from enironment import Environment
|
||||
import requests
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class UrlCheck(AbstractStep[str]):
|
||||
def __init__(self, env: Environment, url: str, expected: Any, **kwargs):
|
||||
|
||||
def __init__(self, env: Environment, url: str, expected: Any, **kwargs): # type: ignore[no-untyped-def]
|
||||
super().__init__(env=env, **kwargs)
|
||||
self.url = url
|
||||
self.expected = expected
|
||||
@@ -53,7 +47,7 @@ class UrlCheck(AbstractStep[str]):
|
||||
return "Ok"
|
||||
|
||||
class SimpleLog(AbstractStep[Any]):
|
||||
def __init__(self, env: Environment, message: Any, **kwargs):
|
||||
def __init__(self, env: Environment, message: Any, **kwargs):# type: ignore[no-untyped-def]
|
||||
super().__init__(env=env, **kwargs)
|
||||
self.message = message
|
||||
|
||||
|
||||
+20
-15
@@ -2,9 +2,10 @@ import os
|
||||
import subprocess
|
||||
import logging
|
||||
import docker
|
||||
from docker import errors as docker_errors
|
||||
import yaml
|
||||
from dotenv import dotenv_values
|
||||
from typing import List, Optional, Dict, Callable, Any
|
||||
from typing import List, Dict, Callable, Any
|
||||
from dataclasses import dataclass
|
||||
from steps.step import AbstractStep
|
||||
from steps.git import GitClone
|
||||
@@ -14,13 +15,13 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
class DockerComposeBuild(AbstractStep[List[str]]):
|
||||
def __init__(self,
|
||||
wd: GitClone,
|
||||
docker_repo_username:str,
|
||||
docker_repo_password:str,
|
||||
docker_compose_path:str,
|
||||
docker_repo_url: str,
|
||||
publish: bool,
|
||||
envs: Callable[[], Dict[str, str]], **kwargs):
|
||||
wd: GitClone,
|
||||
docker_repo_username: str,
|
||||
docker_repo_password: str,
|
||||
docker_compose_path: str,
|
||||
docker_repo_url: str,
|
||||
publish: bool,
|
||||
envs: Callable[[], Dict[str, Any]], **kwargs: Any) -> None:
|
||||
super().__init__(**kwargs)
|
||||
self.wd = wd
|
||||
self.envs = envs
|
||||
@@ -31,7 +32,6 @@ class DockerComposeBuild(AbstractStep[List[str]]):
|
||||
self.publish = publish
|
||||
|
||||
def progress(self) -> List[str]:
|
||||
# def process(self, docker_compose_path: str, docker_repo_url: str, docker_repo_username: str, docker_repo_password: str) -> DockerBuildResult:
|
||||
"""
|
||||
Build and push Docker images defined in a docker-compose file.
|
||||
"""
|
||||
@@ -40,7 +40,11 @@ class DockerComposeBuild(AbstractStep[List[str]]):
|
||||
# Authenticate to docker repo
|
||||
client = docker.DockerClient(base_url='unix://var/run/docker.sock')
|
||||
if self.publish:
|
||||
client.login(username=self.docker_repo_username, password=self.docker_repo_password, registry=self.docker_repo_url)
|
||||
client.login(
|
||||
username=self.docker_repo_username,
|
||||
password=self.docker_repo_password,
|
||||
registry=self.docker_repo_url
|
||||
)
|
||||
env = self.envs()
|
||||
# Parse docker-compose file
|
||||
docker_compose_absolute_path = os.path.join(self.wd.result, self.docker_compose_path)
|
||||
@@ -71,7 +75,7 @@ class DockerComposeBuild(AbstractStep[List[str]]):
|
||||
client.images.get(image)
|
||||
logger.info(f"Image {image} already exists locally, skipping build.")
|
||||
continue
|
||||
except docker.errors.ImageNotFound:
|
||||
except docker_errors.ImageNotFound:
|
||||
pass
|
||||
|
||||
logger.info(f"Building image {image} from {build_ctx}")
|
||||
@@ -98,7 +102,7 @@ class DockerSwarmCheck(AbstractStep[Dict[str, DockerSwarmCheckResult]]):
|
||||
|
||||
def __init__(self,
|
||||
stack_name: str,
|
||||
**kwargs):
|
||||
**kwargs: Any) -> None:
|
||||
super().__init__(**kwargs)
|
||||
self.stack_name = stack_name
|
||||
|
||||
@@ -129,7 +133,7 @@ class DockerSwarmDeploy(AbstractStep[str]):
|
||||
docker_compose_path: str,
|
||||
envs: Callable[[], Dict[str, Any]],
|
||||
stack_name: str,
|
||||
**kwargs):
|
||||
**kwargs: Any) -> None:
|
||||
super().__init__(**kwargs)
|
||||
self.wd = wd
|
||||
self.buildDocker = buildDocker
|
||||
@@ -151,7 +155,7 @@ class DockerSwarmDeploy(AbstractStep[str]):
|
||||
raise self.stackChecker.result
|
||||
|
||||
|
||||
def merge_dicts(a, b):
|
||||
def merge_dicts(a: Dict[str, Any], b: Dict[str, Any]) -> None:
|
||||
for k, v in b.items():
|
||||
if (
|
||||
k in a
|
||||
@@ -184,7 +188,8 @@ class DockerSwarmDeploy(AbstractStep[str]):
|
||||
ok = []
|
||||
for svc_name, svc in expected_services.items():
|
||||
expected_image = svc.get("image")
|
||||
running_image = current_services.get(svc_name).image if svc_name in current_services else None
|
||||
running_service = current_services.get(svc_name)
|
||||
running_image = running_service.image if running_service is not None else None
|
||||
l = {
|
||||
"service": svc_name,
|
||||
"expected_image": expected_image,
|
||||
|
||||
+33
-30
@@ -13,7 +13,7 @@ from enironment import Environment
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class GitClone(AbstractStep[str]):
|
||||
def __init__(self, env: Environment, branchNamePrefix="", credEnvPrefix="GIT", **kwargs):
|
||||
def __init__(self, env: Environment, branchNamePrefix: str="", credEnvPrefix: str="GIT", **kwargs: Any):
|
||||
super().__init__(env, **kwargs)
|
||||
self.temp_dir = os.path.join(tempfile.gettempdir(), f"{self.env.id}_{hashlib.sha1(self.env.repo.encode()).hexdigest()[:5]}")
|
||||
self.branchNamePrefix = branchNamePrefix
|
||||
@@ -58,7 +58,7 @@ class GitClone(AbstractStep[str]):
|
||||
|
||||
def get_branches(self) -> Dict[str, List[Any]]:
|
||||
repo = git.Repo(self.temp_dir)
|
||||
result = {}
|
||||
result: Dict[str, List[Any]] = {}
|
||||
for ref in repo.refs:
|
||||
if ref.name.startswith('origin/') and not ref.name.startswith('origin/HEAD'):
|
||||
branch_name = ref.name [len('origin/'):]
|
||||
@@ -89,10 +89,10 @@ class CheckoutMerged(AbstractStep[CheckoutAndMergeResult]):
|
||||
wd: GitClone
|
||||
|
||||
def __init__(self, wd: GitClone,
|
||||
git_user_email,
|
||||
git_user_name,
|
||||
git_user_email: str,
|
||||
git_user_name: str,
|
||||
push: bool = True,
|
||||
**kwargs):
|
||||
**kwargs: Any):
|
||||
super().__init__(**kwargs)
|
||||
self.wd = wd
|
||||
self.git_user_email = git_user_email
|
||||
@@ -122,7 +122,7 @@ class CheckoutMerged(AbstractStep[CheckoutAndMergeResult]):
|
||||
return visited
|
||||
|
||||
def progress(self) -> CheckoutAndMergeResult:
|
||||
if self.wd.result_obj is BaseException:
|
||||
if isinstance(self.wd.result_obj, BaseException):
|
||||
raise self.wd.result_obj
|
||||
|
||||
repo_path = self.wd.result_obj
|
||||
@@ -153,7 +153,7 @@ class CheckoutMerged(AbstractStep[CheckoutAndMergeResult]):
|
||||
|
||||
tree = self._commits_tree(repo)
|
||||
|
||||
merge_commit = [(c, self._find_merge_childs(tree, c)) for c in commit_ids.keys()]
|
||||
merge_commit: list[tuple[Commit, list[Commit]]] = [(c, self._find_merge_childs(tree, c)) for c in commit_ids.keys()]
|
||||
result = set(merge_commit[0][1])
|
||||
for c, l in merge_commit[1:]:
|
||||
result = set(result).intersection(set(l))
|
||||
@@ -161,32 +161,32 @@ class CheckoutMerged(AbstractStep[CheckoutAndMergeResult]):
|
||||
break
|
||||
|
||||
if len(result) > 0:
|
||||
merge_commit = [c for c in merge_commit[0][1] if c in result][0]
|
||||
merge_commit1 = [c for c in merge_commit[0][1] if c in result][0]
|
||||
logger.info(f"Common commit found {merge_commit}")
|
||||
else:
|
||||
merge_commit = None
|
||||
merge_commit1 = None
|
||||
logger.info(f"Common commit not found")
|
||||
|
||||
sorted_commits = sorted(commit_ids.keys(), key=lambda x: x.hexsha)
|
||||
auto_branch_hash = hashlib.sha1(''.join([x.hexsha for x in sorted_commits]).encode()).hexdigest()
|
||||
auto_branch_name = f"auto/{auto_branch_hash}"
|
||||
version = '-'.join([x.hexsha[0:8] for x in sorted_commits])
|
||||
if merge_commit is not None:
|
||||
if merge_commit1 is not None:
|
||||
for head in repo.branches:
|
||||
if head.is_remote and head.commit.hexsha == merge_commit.hexsha:
|
||||
logger.info(f"Merge commit {merge_commit} corresponds to branch {head}")
|
||||
if head.is_remote and head.commit.hexsha == merge_commit1.hexsha: # type: ignore[truthy-function]
|
||||
logger.info(f"Merge commit {merge_commit1} corresponds to branch {head}")
|
||||
repo.git.checkout(head.name)
|
||||
return CheckoutAndMergeResult(head.name, merge_commit.hexsha, version)
|
||||
return CheckoutAndMergeResult(head.name, merge_commit1.hexsha, version)
|
||||
|
||||
|
||||
if merge_commit is not None:
|
||||
repo.git.branch('-f', auto_branch_name, merge_commit.hexsha)
|
||||
if merge_commit1 is not None:
|
||||
repo.git.branch('-f', auto_branch_name, merge_commit1.hexsha)
|
||||
repo.git.checkout(auto_branch_name)
|
||||
logger.info(f"Head is {repo.head.commit}")
|
||||
if self.push:
|
||||
repo.git.push('-f', 'origin', auto_branch_name)
|
||||
logger.info(f"Pushed {merge_commit} to {auto_branch_name}")
|
||||
return CheckoutAndMergeResult(auto_branch_name, merge_commit.hexsha, version)
|
||||
return CheckoutAndMergeResult(auto_branch_name, merge_commit1.hexsha, version)
|
||||
|
||||
logger.info(f"Looking for existing auto branch: {auto_branch_name}")
|
||||
|
||||
@@ -233,30 +233,33 @@ class CheckoutMerged(AbstractStep[CheckoutAndMergeResult]):
|
||||
|
||||
from steps.docker import DockerSwarmCheckResult, DockerSwarmCheck
|
||||
|
||||
class GitUnmerge(AbstractStep[List[str]]):
|
||||
class GitUnmerge(AbstractStep[List[Tuple[str, str]]]):
|
||||
wd: GitClone
|
||||
|
||||
def __init__(self, wd: GitClone,
|
||||
check: DockerSwarmCheck,
|
||||
**kwargs):
|
||||
**kwargs: Any):
|
||||
super().__init__(**kwargs)
|
||||
self.wd = wd
|
||||
self.check = check
|
||||
|
||||
def progress(self) -> List[str]:
|
||||
def progress(self) -> List[Tuple[str, str]]:
|
||||
wd = self.wd.result
|
||||
deployState: Dict[str, DockerSwarmCheckResult] = self.check.result
|
||||
|
||||
version = set([it.version for it in deployState.values()])
|
||||
if len(version) != 1:
|
||||
raise BaseException(f"Expected exactly one version, got: {version}")
|
||||
version = list(version)[0]
|
||||
versions = set([it.version for it in deployState.values()])
|
||||
if len(versions) != 1:
|
||||
raise BaseException(f"Expected exactly one version, got: {versions}")
|
||||
|
||||
version = list(versions)[0]
|
||||
if version is None:
|
||||
raise BaseException(f"Expected exactly one version, got: {versions}")
|
||||
repo = git.Repo(wd)
|
||||
if 'auto-' in version:
|
||||
version = version[len('auto-'):].split('-')
|
||||
version_parts = version[len('auto-'):].split('-')
|
||||
|
||||
commits = []
|
||||
for v in version:
|
||||
commits: List[Tuple[str, str]] = []
|
||||
for v in version_parts:
|
||||
commit = repo.commit(v)
|
||||
# branches = [head.name for head in repo.heads if head.commit.hexsha == commit.hexsha]
|
||||
branches = [ref.name for ref in repo.remotes.origin.refs if ref.commit.hexsha == commit.hexsha]
|
||||
@@ -266,10 +269,10 @@ class GitUnmerge(AbstractStep[List[str]]):
|
||||
# # Find all parent commits of the given commit
|
||||
# parents = [parent for parent in commit.parents]
|
||||
# branches = [head.name for head in repo.heads for commit in parents if head.commit.hexsha == commit.hexsha]
|
||||
commits.append((commit.hexsha, branches))
|
||||
for b in branches:
|
||||
commits.append((b, commit.hexsha))
|
||||
|
||||
version = commits
|
||||
return commits
|
||||
else:
|
||||
version = [version]
|
||||
return version
|
||||
raise BaseException(f"Version format not recognized: {version}")
|
||||
|
||||
|
||||
@@ -4,10 +4,8 @@ import git
|
||||
import logging
|
||||
import subprocess
|
||||
import tempfile
|
||||
import json
|
||||
import requests
|
||||
from dataclasses import dataclass, field
|
||||
from typing import List, Optional, Dict, Any
|
||||
from typing import List, Optional, Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -19,7 +17,7 @@ class GradleBuildResult:
|
||||
error_message: Optional[str] = None
|
||||
|
||||
class GradleBuild:
|
||||
def process(self, checkout_result, task_name: str) -> GradleBuildResult:
|
||||
def process(self, checkout_result: Any, task_name: str) -> GradleBuildResult:
|
||||
"""
|
||||
Build the Gradle project and push Docker images.
|
||||
|
||||
@@ -81,7 +79,7 @@ class GradleBuild:
|
||||
artifact_urls=artifact_urls,
|
||||
success=True
|
||||
)
|
||||
|
||||
|
||||
except Exception as e:
|
||||
error_message = f"Gradle build failed: {str(e)}"
|
||||
logger.error(error_message)
|
||||
@@ -118,13 +116,13 @@ class GradleBuild:
|
||||
|
||||
with open(properties_file, 'w') as f:
|
||||
f.write(content)
|
||||
|
||||
|
||||
logger.info(f"Updated version to {version} in gradle.properties")
|
||||
|
||||
|
||||
def _extract_docker_images(self, project_dir: str) -> List[str]:
|
||||
"""Extract Docker image names from Gradle modules"""
|
||||
images = []
|
||||
|
||||
|
||||
# Look for build.gradle or build.gradle.kts files
|
||||
for root, dirs, files in os.walk(project_dir):
|
||||
for file in files:
|
||||
@@ -140,13 +138,13 @@ class GradleBuild:
|
||||
images.extend(image_matches)
|
||||
|
||||
return list(set(images)) # Remove duplicates
|
||||
|
||||
|
||||
def _check_images_exist(self, images: List[str], version: str) -> bool:
|
||||
"""Check if Docker images with the given version exist in the registry"""
|
||||
# For demonstration purposes, we'll assume images don't exist
|
||||
# In a real implementation, you would check against your Docker registry
|
||||
return False
|
||||
|
||||
|
||||
def _run_gradle_task(self, project_dir: str, task_name: str) -> GradleBuildResult:
|
||||
"""Run a Gradle task in the project directory"""
|
||||
try:
|
||||
@@ -159,7 +157,7 @@ class GradleBuild:
|
||||
else:
|
||||
# Use global gradle
|
||||
gradle_cmd = ['gradle', task_name]
|
||||
|
||||
|
||||
# Run the command
|
||||
logger.info(f"Running Gradle command: {' '.join(gradle_cmd)}")
|
||||
result = subprocess.run(
|
||||
@@ -168,15 +166,15 @@ class GradleBuild:
|
||||
capture_output=True,
|
||||
text=True
|
||||
)
|
||||
|
||||
|
||||
if result.returncode != 0:
|
||||
return GradleBuildResult(
|
||||
success=False,
|
||||
error_message=f"Gradle task failed: {result.stderr}"
|
||||
)
|
||||
|
||||
|
||||
return GradleBuildResult(success=True)
|
||||
|
||||
|
||||
except Exception as e:
|
||||
return GradleBuildResult(
|
||||
success=False,
|
||||
|
||||
@@ -1,17 +1,20 @@
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import List, Optional, Union, Tuple, Any
|
||||
from typing import Union
|
||||
from typing import TypeVar, Generic
|
||||
from dataclasses import dataclass, asdict, field
|
||||
from enironment import Environment
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class NotReadyException(Exception):
|
||||
def __init__(self, message: str):
|
||||
super().__init__(message)
|
||||
|
||||
|
||||
T = TypeVar('T')
|
||||
|
||||
|
||||
class AbstractStep(ABC, Generic[T]):
|
||||
|
||||
env: Environment
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
[mypy]
|
||||
# Specify Python version
|
||||
python_version = 3.11
|
||||
|
||||
# Import discovery
|
||||
mypy_path = backend
|
||||
namespace_packages = True
|
||||
explicit_package_bases = True
|
||||
|
||||
# Type checking strictness - require type annotations
|
||||
disallow_untyped_defs = True
|
||||
disallow_incomplete_defs = True
|
||||
check_untyped_defs = True
|
||||
disallow_untyped_decorators = False
|
||||
no_implicit_reexport = True
|
||||
disallow_any_unimported = True
|
||||
disallow_untyped_calls = True
|
||||
# Warnings
|
||||
warn_redundant_casts = True
|
||||
warn_unused_ignores = True
|
||||
warn_return_any = False
|
||||
warn_unreachable = False
|
||||
|
||||
# Error output
|
||||
show_error_codes = True
|
||||
show_column_numbers = True
|
||||
pretty = True
|
||||
|
||||
# Miscellaneous - relaxed for initial adoption
|
||||
strict_equality = False
|
||||
no_implicit_optional = True
|
||||
|
||||
# Allow some flexibility during transition
|
||||
allow_untyped_calls = True
|
||||
allow_untyped_defs = False
|
||||
allow_incomplete_defs = False
|
||||
|
||||
# Per-module options for third-party libraries without type stubs
|
||||
[mypy-git.*]
|
||||
ignore_missing_imports = True
|
||||
|
||||
[mypy-docker.*]
|
||||
ignore_missing_imports = True
|
||||
|
||||
[mypy-flask_socketio.*]
|
||||
ignore_missing_imports = True
|
||||
|
||||
[mypy-dotenv.*]
|
||||
ignore_missing_imports = True
|
||||
|
||||
[mypy-yaml.*]
|
||||
ignore_missing_imports = True
|
||||
|
||||
[mypy-flask.*]
|
||||
ignore_missing_imports = True
|
||||
|
||||
[mypy-socketio.*]
|
||||
ignore_missing_imports = True
|
||||
@@ -0,0 +1,13 @@
|
||||
[project]
|
||||
name = "brencher"
|
||||
version = "0.1.0"
|
||||
description = "Brencher project"
|
||||
requires-python = ">=3.9"
|
||||
|
||||
[tool.pyright]
|
||||
reportMissingModuleSource = false
|
||||
|
||||
[tool.ty]
|
||||
# See https://github.com/astral-sh/ty for more options
|
||||
|
||||
rules.unused-ignore-comment = "ignore"
|
||||
@@ -5,3 +5,9 @@ python-dotenv
|
||||
docker
|
||||
PyYAML
|
||||
websocket-client
|
||||
mypy
|
||||
ty
|
||||
|
||||
types-PyYAML
|
||||
types-requests
|
||||
types-flask-socketio
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
def test_dummy() -> None:
|
||||
assert True
|
||||
Reference in New Issue
Block a user