diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..c2c8fe3 --- /dev/null +++ b/.flake8 @@ -0,0 +1,9 @@ +[flake8] +exclude = + .venv, + __pycache__, + .git, + .pytest_cache, + *.egg-info +max-line-length = 127 +max-complexity = 10 diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 34184da..dce9430 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -17,25 +17,24 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10", "3.11"] + python-version: ["3.11"] + checker: [ + "flake8 . --count --select=E9,F63,F7,F82 --ignore=F824 --show-source --statistics", + "flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics", + # "ty check backend", + "mypy backend --config-file mypy.ini", + "pytest", + ] steps: - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} + - name: Set up Python ${{ matrix.python-version }}, ${{ matrix.checker }}" uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install flake8 pytest - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Test with pytest - run: | - pytest + python -m pip install flake8 pytest ty + pip install -r requirements.txt + ${{ matrix.checker }} diff --git a/.vscode/extensions.json b/.vscode/extensions.json index fc960fb..4265230 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -1,6 +1,10 @@ { "recommendations": [ "ms-python.vscode-pylance", + "vinirossa.vscode-gitandgithub-pack", + "astral-sh.ty" + "ms-python.mypy-type-checker", + "ms-python.python", "vinirossa.vscode-gitandgithub-pack" ] } \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..97fd984 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,18 @@ +{ + "ty.enable": true + "python.linting.enabled": true, + "python.linting.mypyEnabled": true, + "python.linting.mypyArgs": [ + "--config-file=mypy.ini" + ], + "python.analysis.typeCheckingMode": "basic", + "python.analysis.diagnosticMode": "workspace", + "[python]": { + "editor.formatOnSave": false, + "editor.codeActionsOnSave": { + "source.organizeImports": "explicit" + } + }, + "mypy.configFile": "mypy.ini", + "mypy.runUsingActiveInterpreter": true +} diff --git a/README.instruction.md b/README.instruction.md index e9d62f5..04c3e74 100644 --- a/README.instruction.md +++ b/README.instruction.md @@ -19,10 +19,12 @@ Generate docker compose file to deploy to docker swarm. python ./backend/app.py brencher_local1 noweb +# Local start + +python ./backend/app.py brencher_local1 noweb + # Update docker secret - - docker service scale brencher_brencher-backend=0 brencher2_brencher-backend=0 docker service update --secret-rm brencher-secrets brencher_brencher-backend docker service update --secret-rm brencher-secrets brencher2_brencher-backend diff --git a/backend/app.py b/backend/app.py index 802d5e8..f01ddac 100644 --- a/backend/app.py +++ b/backend/app.py @@ -1,25 +1,21 @@ -from multiprocessing import Pipe import os import json import time import threading -import hashlib -from flask import Flask, request, jsonify, send_from_directory +from flask import Flask, send_from_directory from steps.step import AbstractStep from flask_socketio import SocketIO, emit import socketio as socketio_client from dotenv import load_dotenv -import git import enironment from steps.git import GitClone -import tempfile import logging -from dataclasses import dataclass, asdict, field +from dataclasses import asdict from typing import List, Dict, Any, Optional, Tuple -import shutil -import subprocess import traceback from typing import TypeVar +import types +from flask.json.provider import DefaultJSONProvider # Configure logging logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') @@ -30,8 +26,9 @@ logger = logging.getLogger(__name__) load_dotenv("local.env") load_dotenv('/run/secrets/brencher-secrets') + class DataclassJSONEncoder(json.JSONEncoder): - def default(self, o): + def default(self, o: Any) -> Any: if hasattr(o, '__dataclass_fields__'): return asdict(o) if isinstance(o, BaseException): @@ -41,17 +38,16 @@ class DataclassJSONEncoder(json.JSONEncoder): except TypeError: return str(o) -import types + custom_json = types.SimpleNamespace() custom_json.dumps = lambda obj, **kwargs: json.dumps(obj, cls=DataclassJSONEncoder, **kwargs) custom_json.loads = json.loads -from flask.json.provider import DefaultJSONProvider class DataclassJSONProvider(DefaultJSONProvider): - def dumps(self, obj, **kwargs): + def dumps(self, obj: Any, **kwargs: Any) -> str: return json.dumps(obj, cls=DataclassJSONEncoder, **kwargs) - def loads(self, s, **kwargs): + def loads(self, s: str | bytes, **kwargs: Any) -> Any: return json.loads(s, **kwargs) app = Flask(__name__, static_folder='frontend') @@ -63,20 +59,19 @@ FRONTEND_DIR = os.path.join(os.path.dirname(__file__), '../frontend') # In-memory state environments: Dict[str, Tuple[enironment.Environment, List[AbstractStep]]] = {} environments_slaves: Dict[str, Tuple[enironment.Environment, List[AbstractStep]]] = {} -branches = {} -branches_slaves = {} +branches: Dict[str, Dict[str, Any]] = {} +branches_slaves: Dict[str, Dict[str, Any]] = {} state_lock = threading.Lock() @app.route('/') -def serve_index(): +def serve_index() -> Any: return send_from_directory(FRONTEND_DIR, 'index.html') @app.route('/') -def serve_static(path): +def serve_static(path: str) -> Any: return send_from_directory(FRONTEND_DIR, path) # --- WebSocket Endpoints --- -from flask import copy_current_request_context from flask_socketio import Namespace T = TypeVar('T') @@ -84,26 +79,25 @@ def merge_dicts(a: Dict[str, T], b: Dict[str, T]) -> Dict[str, T]: result: Dict[str, T] = {} for k in (a.keys() | b.keys()): if k not in a or k not in b: - result[k] = a.get(k, b.get(k)) # type: ignore[arg-type] + result[k] = a.get(k, b.get(k)) # type: ignore[assignment] continue if ( isinstance(a[k], dict) and isinstance(b[k], dict) ): - result[k] = merge_dicts(a[k], b[k]) # type: ignore[arg-type] + result[k] = merge_dicts(a[k], b[k]) # type: ignore[arg-type,assignment] else: result[k] = b[k] return result slave_url = os.getenv('SLAVE_BRENCHER') -remote_sio: socketio_client.Client | None = None if slave_url: logger.info(f"SLAVE_BRENCHER set, will connect to master at {slave_url}") remote_sio = socketio_client.Client(logger=False, engineio_logger=False) # Handlers for events from master -> re-emit locally (marked so we don't loop) @remote_sio.on('branches', namespace='/ws/branches') - def _remote_branches(data): + def _remote_branches(data: Any) -> None: global branches, branches_slaves branches_slaves = data merge_result = merge_dicts(branches, branches_slaves) @@ -113,7 +107,7 @@ if slave_url: logger.error(f"Error forwarding remote branches locally: {e}") @remote_sio.on('environments', namespace='/ws/environment') - def _remote_environments(data): + def _remote_environments(data: Any) -> None: global environments, environments_slaves environments_slaves = data try: @@ -122,32 +116,32 @@ if slave_url: logger.error(f"Error forwarding remote environments locally: {e}") @remote_sio.on('error', namespace='/ws/errors') - def _remote_error(data): + def _remote_error(data: Any) -> None: try: socketio.emit('error', data, namespace='/ws/errors') except Exception as e: logger.error(f"Error forwarding remote errors locally: {e}") @remote_sio.event - def connect(): + def connect() -> None: logger.info("Connected to master brencher (SLAVE mode).") @remote_sio.event - def disconnect(): + def disconnect() -> None: logger.info("Disconnected from master brencher (SLAVE mode).") class BranchesNamespace(Namespace): - def on_connect(self, auth=None): + def on_connect(self, auth: Optional[Any] = None) -> None: emit('branches', merge_dicts(branches, branches_slaves)) - def on_update(self, data): + def on_update(self, data: Any) -> None: emit('branches', merge_dicts(branches, branches_slaves)) -def get_local_envs_to_emit() -> Dict[str, Tuple[enironment.Environment, List[AbstractStep]]]: - env_dtos = {} +def get_local_envs_to_emit() -> Dict[str, Tuple[Dict[str, Any], List[Dict[str, Any]]]]: + env_dtos: Dict[str, Tuple[Dict[str, Any], List[Dict[str, Any]]]] = {} for e, p in environments.values(): env = asdict(e) - res = [] + res: List[Dict[str, Any]] = [] for r in p: if isinstance(r.result_obj, BaseException): stack = traceback.format_exception(type(r.result_obj), r.result_obj, r.result_obj.__traceback__) @@ -163,12 +157,12 @@ def get_local_envs_to_emit() -> Dict[str, Tuple[enironment.Environment, List[Abs env_dtos[env['id']] = (env, res) return env_dtos -def get_global_envs_to_emit(): +def get_global_envs_to_emit() -> Any: global environments, environments_slaves local_envs = get_local_envs_to_emit() - merge_result = merge_dicts(local_envs, environments_slaves) - logger.info(f"Local keys: {local_envs.keys()}") - logger.info(f"Slave keys: {environments_slaves.keys()}") + merge_result = merge_dicts(local_envs, environments_slaves) # type: ignore[misc] + # logger.info(f"Local keys: {local_envs.keys()}") + # logger.info(f"Slave keys: {environments_slaves.keys()}") common_keys = set(local_envs.keys()) & set(environments_slaves.keys()) if len(common_keys) > 0: socketio.emit('error', {'message': f"Conflict: both master and slave have environment with id {common_keys}"}, namespace='/ws/errors') @@ -177,15 +171,15 @@ def get_global_envs_to_emit(): environment_update_event = threading.Event() @app.route('/state') -def serve_state(): +def serve_state() -> Any: return get_global_envs_to_emit() class EnvironmentNamespace(Namespace): - def on_connect(self, auth=None): + def on_connect(self, auth: Optional[Any] = None) -> None: emit('environments', get_global_envs_to_emit()) - def on_update(self, data): + def on_update(self, data: Dict[str, Any]) -> None: global environments, remote_sio logger.info(f"Received environment update: {data}") @@ -204,9 +198,9 @@ class EnvironmentNamespace(Namespace): emit('environments', get_global_envs_to_emit(), namespace='/ws/environment') class ErrorsNamespace(Namespace): - def on_connect(self, auth=None): + def on_connect(self, auth: Optional[Any] = None) -> None: pass - def on_error(self, data): + def on_error(self, data: Any) -> None: emit('error', data) socketio.on_namespace(BranchesNamespace('/ws/branches')) @@ -219,7 +213,7 @@ if __name__ == '__main__': import os import sys - def sigchld_handler(signum, frame): + def sigchld_handler(signum, frame): # type: ignore[no-untyped-def] """Reap zombie processes""" while True: try: @@ -249,20 +243,21 @@ if __name__ == '__main__': environments = {e[0].id: e for e in environments_l} import sys - cli_env_ids = sys.argv[1:] - if len(cli_env_ids) == 0: - cli_env_ids = os.getenv('PROFILES', '') + cli_env_ids_list = sys.argv[1:] + cli_env_ids_str: str + if len(cli_env_ids_list) == 0: + cli_env_ids_str = os.getenv('PROFILES', '') else: - cli_env_ids = cli_env_ids[0] - logger.info(f"cli_env_ids {cli_env_ids}") - if len(cli_env_ids) > 0 and cli_env_ids[0] == '-': - cli_env_ids = cli_env_ids[1:].split(',') + cli_env_ids_str = cli_env_ids_list[0] + logger.info(f"cli_env_ids {cli_env_ids_str}") + if len(cli_env_ids_str) > 0 and cli_env_ids_str[0] == '-': + cli_env_ids = cli_env_ids_str[1:].split(',') cli_env_ids = [x for x in cli_env_ids if len(x) > 0] logger.info(f"cli_env_ids (minus) {cli_env_ids}") if cli_env_ids and len(cli_env_ids) > 0: environments = { k: e for k, e in environments.items() if k not in cli_env_ids } else: - cli_env_ids = cli_env_ids.split(',') + cli_env_ids = cli_env_ids_str.split(',') cli_env_ids = [x for x in cli_env_ids if len(x) > 0] logger.info(f"cli_env_ids {cli_env_ids}") if cli_env_ids and len(cli_env_ids) > 0: @@ -275,7 +270,7 @@ if __name__ == '__main__': logger.info(f"Resulting profiles {environments.keys()}") # Background thread to refresh branches every 5 minutes - def emit_fresh_branches(): + def emit_fresh_branches() -> None: global branches, branches_slaves, environments for k, e in environments.items(): env, pipe = e @@ -294,10 +289,10 @@ if __name__ == '__main__': socketio.emit('branches', merge_dicts(branches, branches_slaves), namespace='/ws/branches') - def processing_thread(): + def processing_thread() -> None: while True: import processing - def emit_envs(): + def emit_envs() -> None: try: socketio.emit('environments', get_global_envs_to_emit(), namespace='/ws/environment') except Exception as e: @@ -316,7 +311,7 @@ if __name__ == '__main__': # Connect in background so server startup isn't blocked - def _connect_remote(): + def _connect_remote() -> None: global remote_sio print("Connecting to SLAVE_BRENCHER...") while True: diff --git a/backend/configs/brencher.py b/backend/configs/brencher.py index 910e02b..b20baba 100644 --- a/backend/configs/brencher.py +++ b/backend/configs/brencher.py @@ -2,7 +2,7 @@ from steps.checks import SimpleLog, UrlCheck from steps.git import GitClone, CheckoutMerged, GitUnmerge from steps.docker import DockerComposeBuild, DockerSwarmCheck, DockerSwarmDeploy from enironment import Environment -from typing import List, Dict, Any, Optional, Tuple +from typing import List, Tuple from steps.step import AbstractStep from configs.brencher2 import checkPingF diff --git a/backend/configs/brencher2.py b/backend/configs/brencher2.py index 7d002fa..6f83890 100644 --- a/backend/configs/brencher2.py +++ b/backend/configs/brencher2.py @@ -3,7 +3,7 @@ from steps.checks import SimpleLog, UrlCheck from steps.git import GitClone, CheckoutMerged, GitUnmerge from steps.docker import DockerComposeBuild, DockerSwarmCheck, DockerSwarmDeploy from enironment import Environment -from typing import List, Dict, Any, Optional, Tuple +from typing import List, Any, Tuple from steps.step import AbstractStep env = Environment( @@ -13,7 +13,7 @@ env = Environment( repo="https://github.com/rudolf1/brencher.git", ) -def checkPingF(obj: Any): +def checkPingF(obj: Any) -> None: if not isinstance(obj, dict): raise TypeError(f"Expected dict, got {type(obj).__name__}") if "brencher" not in obj or "brencher2" not in obj: diff --git a/backend/configs/brencher_local1.py b/backend/configs/brencher_local1.py index c275c2b..ebfab65 100644 --- a/backend/configs/brencher_local1.py +++ b/backend/configs/brencher_local1.py @@ -2,7 +2,7 @@ from steps.git import GitClone, CheckoutMerged, GitUnmerge from steps.docker import DockerComposeBuild from steps.docker import DockerSwarmDeploy, DockerSwarmCheck from enironment import Environment -from typing import List, Dict, Any, Optional, Tuple +from typing import List, Tuple from steps.step import AbstractStep env_local = Environment( diff --git a/backend/configs/brencher_local2.py b/backend/configs/brencher_local2.py index 54517f5..9a8fd2b 100644 --- a/backend/configs/brencher_local2.py +++ b/backend/configs/brencher_local2.py @@ -2,7 +2,7 @@ from steps.git import GitClone, CheckoutMerged, GitUnmerge from steps.docker import DockerComposeBuild from steps.docker import DockerSwarmDeploy, DockerSwarmCheck from enironment import Environment -from typing import List, Dict, Any, Optional, Tuple +from typing import List, Tuple from steps.step import AbstractStep env_local = Environment( diff --git a/backend/configs/immich.py b/backend/configs/immich.py index cfd5f46..6e16b6c 100644 --- a/backend/configs/immich.py +++ b/backend/configs/immich.py @@ -1,7 +1,7 @@ from steps.git import GitClone, CheckoutMerged, GitUnmerge -from steps.docker import DockerComposeBuild, DockerSwarmCheck, DockerSwarmDeploy +from steps.docker import DockerSwarmCheck, DockerSwarmDeploy from enironment import Environment -from typing import List, Dict, Any, Optional, Tuple +from typing import List, Tuple from steps.step import AbstractStep from steps.checks import SimpleLog, UrlCheck diff --git a/backend/configs/torrserv_proxy.py b/backend/configs/torrserv_proxy.py index f86d2a7..446deda 100644 --- a/backend/configs/torrserv_proxy.py +++ b/backend/configs/torrserv_proxy.py @@ -1,7 +1,7 @@ from steps.git import GitClone, CheckoutMerged, GitUnmerge from steps.docker import DockerComposeBuild, DockerSwarmCheck, DockerSwarmDeploy from enironment import Environment -from typing import List, Dict, Any, Optional, Tuple +from typing import List, Tuple from steps.step import AbstractStep env = Environment( diff --git a/backend/enironment.py b/backend/enironment.py index e46e031..3aa7e55 100644 --- a/backend/enironment.py +++ b/backend/enironment.py @@ -1,5 +1,5 @@ -from dataclasses import dataclass, asdict, field -from typing import List, Dict, Any, Optional, Tuple, Callable +from dataclasses import dataclass +from typing import List, Tuple @dataclass class Environment: diff --git a/backend/processing.py b/backend/processing.py index 0727562..7453acb 100644 --- a/backend/processing.py +++ b/backend/processing.py @@ -1,7 +1,7 @@ from enironment import Environment from steps.step import AbstractStep from steps.git import GitUnmerge -from typing import List, Dict, Any, Optional, Tuple, Callable +from typing import List, Tuple, Callable import logging logger = logging.getLogger(__name__) @@ -9,7 +9,7 @@ logger = logging.getLogger(__name__) def process_all_jobs( environemnts: List[Tuple[Environment, List[AbstractStep]]], onupdate: Callable[[], None] - ): + ) -> None: for env, pipe in environemnts: for step in pipe: try: @@ -19,7 +19,7 @@ def process_all_jobs( # TODO Move to separate job. # If branches list empty, need to find any brunch which includes commit and add pair (branch, commit) # If branches not empty, need to find most priority branch (project specific) and add (branch, HEAD) - env.branches = [ (b1, c) for c,b in step.result for b1 in b ] + env.branches= step.result logger.error(f"Branches on startup resolved {env.id}, job {step.name}: {env.branches}") except BaseException as e: error_msg = f"Error processing release {env.id}, job {step.name}: {str(e)}" diff --git a/backend/steps/checks.py b/backend/steps/checks.py index de95706..dcd7ae9 100644 --- a/backend/steps/checks.py +++ b/backend/steps/checks.py @@ -1,20 +1,14 @@ -import tempfile -import git -from git.objects import Commit -from git.refs import Reference import logging -from dataclasses import dataclass, asdict, field -from typing import List, Optional, Union, Tuple, Set, Iterator, Dict, Any +from typing import Any from steps.step import AbstractStep -import tempfile -import os from enironment import Environment import requests logger = logging.getLogger(__name__) class UrlCheck(AbstractStep[str]): - def __init__(self, env: Environment, url: str, expected: Any, **kwargs): + + def __init__(self, env: Environment, url: str, expected: Any, **kwargs): # type: ignore[no-untyped-def] super().__init__(env=env, **kwargs) self.url = url self.expected = expected @@ -53,7 +47,7 @@ class UrlCheck(AbstractStep[str]): return "Ok" class SimpleLog(AbstractStep[Any]): - def __init__(self, env: Environment, message: Any, **kwargs): + def __init__(self, env: Environment, message: Any, **kwargs):# type: ignore[no-untyped-def] super().__init__(env=env, **kwargs) self.message = message diff --git a/backend/steps/docker.py b/backend/steps/docker.py index a2fc405..263dc05 100644 --- a/backend/steps/docker.py +++ b/backend/steps/docker.py @@ -2,9 +2,10 @@ import os import subprocess import logging import docker +from docker import errors as docker_errors import yaml from dotenv import dotenv_values -from typing import List, Optional, Dict, Callable, Any +from typing import List, Dict, Callable, Any from dataclasses import dataclass from steps.step import AbstractStep from steps.git import GitClone @@ -14,13 +15,13 @@ logger = logging.getLogger(__name__) class DockerComposeBuild(AbstractStep[List[str]]): def __init__(self, - wd: GitClone, - docker_repo_username:str, - docker_repo_password:str, - docker_compose_path:str, - docker_repo_url: str, - publish: bool, - envs: Callable[[], Dict[str, str]], **kwargs): + wd: GitClone, + docker_repo_username: str, + docker_repo_password: str, + docker_compose_path: str, + docker_repo_url: str, + publish: bool, + envs: Callable[[], Dict[str, Any]], **kwargs: Any) -> None: super().__init__(**kwargs) self.wd = wd self.envs = envs @@ -31,7 +32,6 @@ class DockerComposeBuild(AbstractStep[List[str]]): self.publish = publish def progress(self) -> List[str]: - # def process(self, docker_compose_path: str, docker_repo_url: str, docker_repo_username: str, docker_repo_password: str) -> DockerBuildResult: """ Build and push Docker images defined in a docker-compose file. """ @@ -40,7 +40,11 @@ class DockerComposeBuild(AbstractStep[List[str]]): # Authenticate to docker repo client = docker.DockerClient(base_url='unix://var/run/docker.sock') if self.publish: - client.login(username=self.docker_repo_username, password=self.docker_repo_password, registry=self.docker_repo_url) + client.login( + username=self.docker_repo_username, + password=self.docker_repo_password, + registry=self.docker_repo_url + ) env = self.envs() # Parse docker-compose file docker_compose_absolute_path = os.path.join(self.wd.result, self.docker_compose_path) @@ -71,7 +75,7 @@ class DockerComposeBuild(AbstractStep[List[str]]): client.images.get(image) logger.info(f"Image {image} already exists locally, skipping build.") continue - except docker.errors.ImageNotFound: + except docker_errors.ImageNotFound: pass logger.info(f"Building image {image} from {build_ctx}") @@ -98,7 +102,7 @@ class DockerSwarmCheck(AbstractStep[Dict[str, DockerSwarmCheckResult]]): def __init__(self, stack_name: str, - **kwargs): + **kwargs: Any) -> None: super().__init__(**kwargs) self.stack_name = stack_name @@ -129,7 +133,7 @@ class DockerSwarmDeploy(AbstractStep[str]): docker_compose_path: str, envs: Callable[[], Dict[str, Any]], stack_name: str, - **kwargs): + **kwargs: Any) -> None: super().__init__(**kwargs) self.wd = wd self.buildDocker = buildDocker @@ -151,7 +155,7 @@ class DockerSwarmDeploy(AbstractStep[str]): raise self.stackChecker.result - def merge_dicts(a, b): + def merge_dicts(a: Dict[str, Any], b: Dict[str, Any]) -> None: for k, v in b.items(): if ( k in a @@ -184,7 +188,8 @@ class DockerSwarmDeploy(AbstractStep[str]): ok = [] for svc_name, svc in expected_services.items(): expected_image = svc.get("image") - running_image = current_services.get(svc_name).image if svc_name in current_services else None + running_service = current_services.get(svc_name) + running_image = running_service.image if running_service is not None else None l = { "service": svc_name, "expected_image": expected_image, diff --git a/backend/steps/git.py b/backend/steps/git.py index eababcb..69a45bf 100644 --- a/backend/steps/git.py +++ b/backend/steps/git.py @@ -13,7 +13,7 @@ from enironment import Environment logger = logging.getLogger(__name__) class GitClone(AbstractStep[str]): - def __init__(self, env: Environment, branchNamePrefix="", credEnvPrefix="GIT", **kwargs): + def __init__(self, env: Environment, branchNamePrefix: str="", credEnvPrefix: str="GIT", **kwargs: Any): super().__init__(env, **kwargs) self.temp_dir = os.path.join(tempfile.gettempdir(), f"{self.env.id}_{hashlib.sha1(self.env.repo.encode()).hexdigest()[:5]}") self.branchNamePrefix = branchNamePrefix @@ -58,7 +58,7 @@ class GitClone(AbstractStep[str]): def get_branches(self) -> Dict[str, List[Any]]: repo = git.Repo(self.temp_dir) - result = {} + result: Dict[str, List[Any]] = {} for ref in repo.refs: if ref.name.startswith('origin/') and not ref.name.startswith('origin/HEAD'): branch_name = ref.name [len('origin/'):] @@ -89,10 +89,10 @@ class CheckoutMerged(AbstractStep[CheckoutAndMergeResult]): wd: GitClone def __init__(self, wd: GitClone, - git_user_email, - git_user_name, + git_user_email: str, + git_user_name: str, push: bool = True, - **kwargs): + **kwargs: Any): super().__init__(**kwargs) self.wd = wd self.git_user_email = git_user_email @@ -122,7 +122,7 @@ class CheckoutMerged(AbstractStep[CheckoutAndMergeResult]): return visited def progress(self) -> CheckoutAndMergeResult: - if self.wd.result_obj is BaseException: + if isinstance(self.wd.result_obj, BaseException): raise self.wd.result_obj repo_path = self.wd.result_obj @@ -153,7 +153,7 @@ class CheckoutMerged(AbstractStep[CheckoutAndMergeResult]): tree = self._commits_tree(repo) - merge_commit = [(c, self._find_merge_childs(tree, c)) for c in commit_ids.keys()] + merge_commit: list[tuple[Commit, list[Commit]]] = [(c, self._find_merge_childs(tree, c)) for c in commit_ids.keys()] result = set(merge_commit[0][1]) for c, l in merge_commit[1:]: result = set(result).intersection(set(l)) @@ -161,32 +161,32 @@ class CheckoutMerged(AbstractStep[CheckoutAndMergeResult]): break if len(result) > 0: - merge_commit = [c for c in merge_commit[0][1] if c in result][0] + merge_commit1 = [c for c in merge_commit[0][1] if c in result][0] logger.info(f"Common commit found {merge_commit}") else: - merge_commit = None + merge_commit1 = None logger.info(f"Common commit not found") sorted_commits = sorted(commit_ids.keys(), key=lambda x: x.hexsha) auto_branch_hash = hashlib.sha1(''.join([x.hexsha for x in sorted_commits]).encode()).hexdigest() auto_branch_name = f"auto/{auto_branch_hash}" version = '-'.join([x.hexsha[0:8] for x in sorted_commits]) - if merge_commit is not None: + if merge_commit1 is not None: for head in repo.branches: - if head.is_remote and head.commit.hexsha == merge_commit.hexsha: - logger.info(f"Merge commit {merge_commit} corresponds to branch {head}") + if head.is_remote and head.commit.hexsha == merge_commit1.hexsha: # type: ignore[truthy-function] + logger.info(f"Merge commit {merge_commit1} corresponds to branch {head}") repo.git.checkout(head.name) - return CheckoutAndMergeResult(head.name, merge_commit.hexsha, version) + return CheckoutAndMergeResult(head.name, merge_commit1.hexsha, version) - if merge_commit is not None: - repo.git.branch('-f', auto_branch_name, merge_commit.hexsha) + if merge_commit1 is not None: + repo.git.branch('-f', auto_branch_name, merge_commit1.hexsha) repo.git.checkout(auto_branch_name) logger.info(f"Head is {repo.head.commit}") if self.push: repo.git.push('-f', 'origin', auto_branch_name) logger.info(f"Pushed {merge_commit} to {auto_branch_name}") - return CheckoutAndMergeResult(auto_branch_name, merge_commit.hexsha, version) + return CheckoutAndMergeResult(auto_branch_name, merge_commit1.hexsha, version) logger.info(f"Looking for existing auto branch: {auto_branch_name}") @@ -233,30 +233,33 @@ class CheckoutMerged(AbstractStep[CheckoutAndMergeResult]): from steps.docker import DockerSwarmCheckResult, DockerSwarmCheck -class GitUnmerge(AbstractStep[List[str]]): +class GitUnmerge(AbstractStep[List[Tuple[str, str]]]): wd: GitClone def __init__(self, wd: GitClone, check: DockerSwarmCheck, - **kwargs): + **kwargs: Any): super().__init__(**kwargs) self.wd = wd self.check = check - def progress(self) -> List[str]: + def progress(self) -> List[Tuple[str, str]]: wd = self.wd.result deployState: Dict[str, DockerSwarmCheckResult] = self.check.result - version = set([it.version for it in deployState.values()]) - if len(version) != 1: - raise BaseException(f"Expected exactly one version, got: {version}") - version = list(version)[0] + versions = set([it.version for it in deployState.values()]) + if len(versions) != 1: + raise BaseException(f"Expected exactly one version, got: {versions}") + + version = list(versions)[0] + if version is None: + raise BaseException(f"Expected exactly one version, got: {versions}") repo = git.Repo(wd) if 'auto-' in version: - version = version[len('auto-'):].split('-') + version_parts = version[len('auto-'):].split('-') - commits = [] - for v in version: + commits: List[Tuple[str, str]] = [] + for v in version_parts: commit = repo.commit(v) # branches = [head.name for head in repo.heads if head.commit.hexsha == commit.hexsha] branches = [ref.name for ref in repo.remotes.origin.refs if ref.commit.hexsha == commit.hexsha] @@ -266,10 +269,10 @@ class GitUnmerge(AbstractStep[List[str]]): # # Find all parent commits of the given commit # parents = [parent for parent in commit.parents] # branches = [head.name for head in repo.heads for commit in parents if head.commit.hexsha == commit.hexsha] - commits.append((commit.hexsha, branches)) + for b in branches: + commits.append((b, commit.hexsha)) - version = commits + return commits else: - version = [version] - return version + raise BaseException(f"Version format not recognized: {version}") diff --git a/backend/steps/gradle_build.py b/backend/steps/gradle_build.py index c77df30..776371a 100644 --- a/backend/steps/gradle_build.py +++ b/backend/steps/gradle_build.py @@ -4,10 +4,8 @@ import git import logging import subprocess import tempfile -import json -import requests from dataclasses import dataclass, field -from typing import List, Optional, Dict, Any +from typing import List, Optional, Any logger = logging.getLogger(__name__) @@ -19,7 +17,7 @@ class GradleBuildResult: error_message: Optional[str] = None class GradleBuild: - def process(self, checkout_result, task_name: str) -> GradleBuildResult: + def process(self, checkout_result: Any, task_name: str) -> GradleBuildResult: """ Build the Gradle project and push Docker images. @@ -81,7 +79,7 @@ class GradleBuild: artifact_urls=artifact_urls, success=True ) - + except Exception as e: error_message = f"Gradle build failed: {str(e)}" logger.error(error_message) @@ -118,13 +116,13 @@ class GradleBuild: with open(properties_file, 'w') as f: f.write(content) - + logger.info(f"Updated version to {version} in gradle.properties") - + def _extract_docker_images(self, project_dir: str) -> List[str]: """Extract Docker image names from Gradle modules""" images = [] - + # Look for build.gradle or build.gradle.kts files for root, dirs, files in os.walk(project_dir): for file in files: @@ -140,13 +138,13 @@ class GradleBuild: images.extend(image_matches) return list(set(images)) # Remove duplicates - + def _check_images_exist(self, images: List[str], version: str) -> bool: """Check if Docker images with the given version exist in the registry""" # For demonstration purposes, we'll assume images don't exist # In a real implementation, you would check against your Docker registry return False - + def _run_gradle_task(self, project_dir: str, task_name: str) -> GradleBuildResult: """Run a Gradle task in the project directory""" try: @@ -159,7 +157,7 @@ class GradleBuild: else: # Use global gradle gradle_cmd = ['gradle', task_name] - + # Run the command logger.info(f"Running Gradle command: {' '.join(gradle_cmd)}") result = subprocess.run( @@ -168,15 +166,15 @@ class GradleBuild: capture_output=True, text=True ) - + if result.returncode != 0: return GradleBuildResult( success=False, error_message=f"Gradle task failed: {result.stderr}" ) - + return GradleBuildResult(success=True) - + except Exception as e: return GradleBuildResult( success=False, diff --git a/backend/steps/step.py b/backend/steps/step.py index d76ac09..695c7df 100644 --- a/backend/steps/step.py +++ b/backend/steps/step.py @@ -1,17 +1,20 @@ from abc import ABC, abstractmethod -from typing import List, Optional, Union, Tuple, Any +from typing import Union from typing import TypeVar, Generic -from dataclasses import dataclass, asdict, field from enironment import Environment import logging logger = logging.getLogger(__name__) + class NotReadyException(Exception): def __init__(self, message: str): super().__init__(message) + T = TypeVar('T') + + class AbstractStep(ABC, Generic[T]): env: Environment diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 0000000..02befdf --- /dev/null +++ b/mypy.ini @@ -0,0 +1,58 @@ +[mypy] +# Specify Python version +python_version = 3.11 + +# Import discovery +mypy_path = backend +namespace_packages = True +explicit_package_bases = True + +# Type checking strictness - require type annotations +disallow_untyped_defs = True +disallow_incomplete_defs = True +check_untyped_defs = True +disallow_untyped_decorators = False +no_implicit_reexport = True +disallow_any_unimported = True +disallow_untyped_calls = True +# Warnings +warn_redundant_casts = True +warn_unused_ignores = True +warn_return_any = False +warn_unreachable = False + +# Error output +show_error_codes = True +show_column_numbers = True +pretty = True + +# Miscellaneous - relaxed for initial adoption +strict_equality = False +no_implicit_optional = True + +# Allow some flexibility during transition +allow_untyped_calls = True +allow_untyped_defs = False +allow_incomplete_defs = False + +# Per-module options for third-party libraries without type stubs +[mypy-git.*] +ignore_missing_imports = True + +[mypy-docker.*] +ignore_missing_imports = True + +[mypy-flask_socketio.*] +ignore_missing_imports = True + +[mypy-dotenv.*] +ignore_missing_imports = True + +[mypy-yaml.*] +ignore_missing_imports = True + +[mypy-flask.*] +ignore_missing_imports = True + +[mypy-socketio.*] +ignore_missing_imports = True diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..c0fe11e --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,13 @@ +[project] +name = "brencher" +version = "0.1.0" +description = "Brencher project" +requires-python = ">=3.9" + +[tool.pyright] +reportMissingModuleSource = false + +[tool.ty] +# See https://github.com/astral-sh/ty for more options + +rules.unused-ignore-comment = "ignore" \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index b7de2b8..4b36362 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,9 @@ python-dotenv docker PyYAML websocket-client +mypy +ty + +types-PyYAML +types-requests +types-flask-socketio diff --git a/tests/test_dummy.py b/tests/test_dummy.py new file mode 100644 index 0000000..3fb8a97 --- /dev/null +++ b/tests/test_dummy.py @@ -0,0 +1,2 @@ +def test_dummy() -> None: + assert True