diff --git a/backend/app.py b/backend/app.py index 1ff0b8f..0ced9f9 100644 --- a/backend/app.py +++ b/backend/app.py @@ -6,7 +6,7 @@ import threading import traceback from typing import Any, Callable, Dict, List, Optional -from secondary import SecondaryManager +from discover_envs import build_environments, parse_arguments from utils import sigchld_handler from dotenv import load_dotenv @@ -23,67 +23,6 @@ load_dotenv("local.env") load_dotenv('/run/secrets/brencher-secrets') -def build_environments(cli_env_ids_str: str, dry_run: bool = False) -> Dict[str, Environment]: - import configs.brencher - import configs.brencher2 - import configs.brencher_local2 - import configs.brencher_local1 - import configs.torrserv_proxy - import configs.immich - import configs.registry - import configs.gmail_mcp - environments_l: List[Environment] = [ - configs.brencher.brencher, - configs.brencher2.brencher2, - configs.brencher_local2.brencher_local2, - configs.brencher_local1.brencher_local1, - configs.torrserv_proxy.torrserv_proxy, - configs.immich.immich, - configs.registry.registry, - configs.gmail_mcp.gmail_mcp, - ] - environments: Dict[str, Environment] = {e.id: e for e in environments_l} - - logger.info(f"cli_env_ids {cli_env_ids_str}") - if len(cli_env_ids_str) > 0 and cli_env_ids_str[0] == '-': - cli_env_ids = cli_env_ids_str[1:].split(',') - cli_env_ids = [x for x in cli_env_ids if len(x) > 0] - logger.info(f"cli_env_ids (minus) {cli_env_ids}") - if cli_env_ids and len(cli_env_ids) > 0: - environments = {k: e for k, e in environments.items() if k not in cli_env_ids} - else: - cli_env_pairs = {x[0]: x[1] if len(x) > 1 else None for x in [x.split(":") - for x in cli_env_ids_str.split(',') - if len(x) > 0 - ] - } - logger.info(f"cli_env_ids {cli_env_pairs}") - if cli_env_pairs and len(cli_env_pairs) > 0: - environments = {k: e for k, e in environments.items() if k in cli_env_pairs.keys()} - for k, v in cli_env_pairs.items(): - if v is not None: - if k in environments: - env = environments[k] - for step in env.pipeline: - resolve_step = step - if isinstance(step, CachingStep): - resolve_step = step._step - if isinstance(resolve_step, SharedStateHolder): - resolve_step.set_branches([(v, 'HEAD')]) - logger.info(f"Overriding environment {k} branches to {(v, 'HEAD')}") - else: - logger.warning(f"Environment {k} not found to override branches") - else: - logger.info(f"No branch override for environment {k}") - if dry_run: - for e in environments.values(): - e.state.set_dry(True) - - logger.info(f"Resulting profiles {environments.keys()}") - - return environments - - class App: def __init__(self, environments: Dict[str, Environment]) -> None: @@ -188,7 +127,11 @@ if __name__ == '__main__': cli_env_ids_str = cli_env_ids_list[0] signal.signal(signal.SIGCHLD, sigchld_handler) - environments = build_environments(cli_env_ids_str, 'dry' in sys.argv[1:]) + environments = build_environments(cli_env_ids_str) + if 'dry' in sys.argv[1:]: + for e in environments.values(): + e.state.set_dry(True) + app1 = App(environments) if 'noweb' in sys.argv[1:]: diff --git a/backend/discover_envs.py b/backend/discover_envs.py new file mode 100644 index 0000000..1423bea --- /dev/null +++ b/backend/discover_envs.py @@ -0,0 +1,71 @@ +import importlib +import logging +import pkgutil +from dataclasses import dataclass, field +from typing import Dict, List, Optional + +from enironment import Environment, SharedStateHolder +from steps.step import CachingStep + +logger = logging.getLogger(__name__) + +@dataclass +class ParsedEnvArgs: + excludes: List[str] = field(default_factory=list) + includes: Dict[str, Optional[str]] = field(default_factory=dict) + + +def parse_arguments(cli_env_ids_str: str) -> ParsedEnvArgs: + logger.info(f"cli_env_ids {cli_env_ids_str}") + if len(cli_env_ids_str) > 0 and cli_env_ids_str[0] == '-': + excludes = [x for x in cli_env_ids_str[1:].split(',') if len(x) > 0] + logger.info(f"cli_env_ids (minus) {excludes}") + return ParsedEnvArgs(excludes=excludes) + + includes: Dict[str, Optional[str]] = { + x[0]: x[1] if len(x) > 1 else None + for x in [x.split(":") for x in cli_env_ids_str.split(',') if len(x) > 0] + } + logger.info(f"cli_env_ids {includes}") + return ParsedEnvArgs(includes=includes) + + +def _discover_environments() -> Dict[str, Environment]: + import configs + found: Dict[str, Environment] = {} + seen: set[int] = set() + for module_info in pkgutil.iter_modules(configs.__path__, prefix=f"{configs.__name__}."): + module = importlib.import_module(module_info.name) + for value in vars(module).values(): + if isinstance(value, Environment) and id(value) not in seen: + seen.add(id(value)) + found[value.id] = value + return found + + +def build_environments(cli_env_ids_str: str) -> Dict[str, Environment]: + environments: Dict[str, Environment] = _discover_environments() + args: ParsedEnvArgs = parse_arguments(cli_env_ids_str) + + if args.excludes: + environments = {k: e for k, e in environments.items() if k not in args.excludes} + elif args.includes: + environments = {k: e for k, e in environments.items() if k in args.includes} + for k, v in args.includes.items(): + if v is None: + logger.info(f"No branch override for environment {k}") + continue + if k not in environments: + logger.warning(f"Environment {k} not found to override branches") + continue + env = environments[k] + for step in env.pipeline: + resolve_step = step + if isinstance(step, CachingStep): + resolve_step = step._step + if isinstance(resolve_step, SharedStateHolder): + resolve_step.set_branches([(v, 'HEAD')]) + logger.info(f"Overriding environment {k} branches to {(v, 'HEAD')}") + + logger.info(f"Resulting profiles {environments.keys()}") + return environments diff --git a/backend/web.py b/backend/web.py index ec10265..461ffe6 100644 --- a/backend/web.py +++ b/backend/web.py @@ -48,7 +48,8 @@ class WebApp: self.ws_connections: Dict[WebSocket, Dict[str, Any]] = {} self._event_loop: Optional[asyncio.AbstractEventLoop] = None - + self.secondaryManager: Optional[SecondaryManager] = None + @asynccontextmanager async def lifespan(fastapi_app: FastAPI) -> AsyncIterator[None]: self._event_loop = asyncio.get_event_loop()