From ac9bf15e508939b30f610325507c185b79c7f0df Mon Sep 17 00:00:00 2001 From: Petr Vobornik Date: Fri, 17 Feb 2023 20:53:20 +0100 Subject: [PATCH 1/4] refactor: add MrackSession object To replace GlobalContext so that mrack run can be isolated without using global variables. Thus more unit testable and potentially usable in parallel. Signed-off-by: Petr Vobornik --- src/mrack/domain.py | 105 ++++++++++++++++++++++++++++++++++++++ src/mrack/session.py | 117 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 222 insertions(+) create mode 100644 src/mrack/domain.py create mode 100644 src/mrack/session.py diff --git a/src/mrack/domain.py b/src/mrack/domain.py new file mode 100644 index 00000000..fa3af314 --- /dev/null +++ b/src/mrack/domain.py @@ -0,0 +1,105 @@ +# Copyright 2023 Red Hat Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Domain object.""" + +from typing import Dict, Set + +from errors import MrackError + +from mrack.host import Host +from mrack.session import MrackSession + + +class Domain: + """Domain. + + Domain as defined in job metadata file + """ + + _session: MrackSession + _name: str + _type: str + _network: str + _hosts: Set[str] + + def __init__( + self, + session, + name, + type, + network, + hosts: Set[str], + ): + """Initialize domain object.""" + self._session = session + self._name = name + self._type = type + self._network = network + self._hosts = hosts + + def __str__(self): + """Return string representation of domain.""" + host_count = len(self._hosts) + out = f"{self._name} {self._type} {self._network}, hosts: {host_count}" + + return out + + def to_json(self) -> Dict: + """Transform object into representation which is acceptable by `json.dump`.""" + return { + "name": self._name, + "type": self._type, + "network": self._network, + "hosts": self._hosts, + } + + @property + def name(self) -> str: + """Get host provisioning provider.""" + return self._name + + @property + def type(self) -> str: + """Get host operating system.""" + return self._type + + @property + def network(self) -> str: + """Get host group.""" + return self._network + + @property + def hosts(self) -> Set[Host]: + """Get provider host id.""" + hosts: Set[Host] = set() + for host in self._hosts: + try: + hosts.add(self._session.hosts[host]) + except ValueError: + raise MrackError(f"Session is missing host: {host}") + + return hosts + + +def domain_from_json(session: MrackSession, data: Dict) -> Domain: + """Reverse method to Host.__json__() after json.loads().""" + domain = Domain( + session, + data["name"], + data["type"], + data["network"], + data["hosts"], + ) + return domain diff --git a/src/mrack/session.py b/src/mrack/session.py new file mode 100644 index 00000000..5066aef0 --- /dev/null +++ b/src/mrack/session.py @@ -0,0 +1,117 @@ +# Copyright 2023 Red Hat Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Mrack session.""" + +import os +from typing import Dict, Optional + +from mrack.config import MrackConfig, ProvisioningConfig +from mrack.dbdrivers.file import FileDBDriver +from mrack.domain import Domain +from mrack.errors import ConfigError +from mrack.host import Host +from mrack.utils import NoSuchFileHandler, load_yaml + + +class MrackSession: + """A provisioning re-usable session.""" + + _database: FileDBDriver + _mrack_config: MrackConfig + _provisioning_config: ProvisioningConfig + _metadata: Dict + _hosts: Dict[str, Host] + _domains: Dict[str, Domain] + + def __init__(self) -> None: + """Init session.""" + self._metadata = {} + self._hosts = {} + self._domains = {} + + @property + def database(self) -> FileDBDriver: # pylint: disable=invalid-name + """Get FileDBDriver object.""" + return self._database + + @property + def config(self) -> MrackConfig: # pylint: disable=invalid-name + """Get MrackConfig object.""" + return self._mrack_config + + @property + def provisioning_config(self) -> ProvisioningConfig: # pylint: disable=invalid-name + """Get ProvisioningConfig object.""" + return self._provisioning_config + + @property + def metadata(self) -> Dict: # pylint: disable=invalid-name + """Get ProvisioningConfig object.""" + return self._metadata + + @property + def hosts(self) -> Dict[str, Host]: # pylint: disable=invalid-name + """Get dictionary of hosts where key is host name.""" + return self._hosts + + @property + def domains(self) -> Dict[str, Domain]: # pylint: disable=invalid-name + """Get dictionary of domains where key is domain name.""" + return self._domains + + def init( + self, + mrack_config_path: str, + provisioning_config: Optional[ProvisioningConfig] = None, + db_file: Optional[str] = None, + ): + """Initialize MrackSession object with all needed values.""" + self._init_mrack_config(mrack_config_path) + + db_path = db_file or self._mrack_config.db_path(default="./.mrackdb.json") + p_config_path = ( + provisioning_config + or self._mrack_config.provisioning_config_path( + default="./provisioning-config.yaml" + ) + ) + + self._init_db(db_path) + self._init_prov_config(p_config_path) + + def _init_db(self, path: str): + """Initialize file database.""" + self._database = FileDBDriver(path) + + @NoSuchFileHandler(error="Provisioning config file not found: {path}") + def _init_prov_config(self, path: str): + """Load and initialize provisioning configuration.""" + self._provisioning_config = ProvisioningConfig(load_yaml(path)) + + def init_metadata(self, user_defined_path: Optional[str]): + """Load and initialize job metadata.""" + meta_path = user_defined_path or self._mrack_config.metadata_path() + + if not meta_path: + raise ConfigError("Job metadata file path not provided.") + if not os.path.exists(meta_path): + raise ConfigError(f"Job metadata file not found: {meta_path}") + + self._metadata = load_yaml(meta_path) + + def _init_mrack_config(self, mrack_config_path: str): + """Load and initialize mrack configuration.""" + self._mrack_config = MrackConfig(mrack_config_path) + self._mrack_config.load() From 6674e8a5251c3c180a4fabd172fcfa4b62923ef3 Mon Sep 17 00:00:00 2001 From: Petr Vobornik Date: Fri, 17 Feb 2023 21:20:54 +0100 Subject: [PATCH 2/4] refactor: add typing to Host Signed-off-by: Petr Vobornik --- src/mrack/host.py | 53 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 37 insertions(+), 16 deletions(-) diff --git a/src/mrack/host.py b/src/mrack/host.py index 16e7c267..38a0715d 100644 --- a/src/mrack/host.py +++ b/src/mrack/host.py @@ -14,7 +14,11 @@ """Host object.""" +from typing import Dict, List, Optional + from mrack.providers import providers +from mrack.providers.provider import Provider +from mrack.session import MrackSession from mrack.utils import object2json STATUS_PENDING = "pending" @@ -35,11 +39,12 @@ ] -def host_from_json(host_data): +def host_from_json(session, host_data): """Reverse method to Host.__json__() after json.loads().""" provider_name = host_data["provider"] provider = providers.get(provider_name) host = Host( + session, provider, host_data["host_id"], host_data["name"], @@ -62,8 +67,23 @@ class Host: Normalized values from providers to offer consistent interface. """ + _session: MrackSession + _provider: Provider + _host_id: str + _name: str + _operating_system: str + _group: str + _ip_addrs: List[str] + _status: str + _username: Optional[str] + _password: Optional[str] + _rawdata: Optional[Dict] + _error: Optional[Dict] + _meta_extra: Optional[Dict] + def __init__( self, + session, provider, host_id, name, @@ -78,6 +98,7 @@ def __init__( meta_extra=None, ): """Initialize host object.""" + self._session = session self._provider = provider self._host_id = host_id self._name = name @@ -91,7 +112,7 @@ def __init__( self._error = error_obj self._meta_extra = meta_extra - def __str__(self): + def __str__(self) -> str: """Return string representation of host.""" net_str = " ".join(self._ip_addrs) @@ -109,7 +130,7 @@ def __str__(self): ) return out - def to_json(self): + def to_json(self) -> Dict: """Transform object into representation which is acceptable by `json.dump`.""" return { "provider": self._provider.name, @@ -127,47 +148,47 @@ def to_json(self): } @property - def provider(self): + def provider(self) -> Provider: """Get host provisioning provider.""" return self._provider @property - def operating_system(self): + def operating_system(self) -> str: """Get host operating system.""" return self._operating_system @property - def group(self): + def group(self) -> str: """Get host group.""" return self._group @property - def host_id(self): + def host_id(self) -> str: """Get provider host id.""" return self._host_id @property - def name(self): + def name(self) -> str: """Get host name.""" return self._name @property - def ip_addrs(self): + def ip_addrs(self) -> List[str]: """Get host IP addresses.""" return self._ip_addrs @property - def ip_addr(self): + def ip_addr(self) -> str: """Get first host IP address.""" return self._ip_addrs[0] if self._ip_addrs else "" @property - def status(self): + def status(self) -> str: """Get host status.""" return self._status @property - def error(self): + def error(self) -> Optional[Dict]: """Get host error object.""" return self._error @@ -177,21 +198,21 @@ def error(self, value): self._error = value @property - def username(self): + def username(self) -> Optional[str]: """Get username for connecting to host.""" return self._username @property - def password(self): + def password(self) -> Optional[str]: """Get password for connecting to host.""" return self._password @property - def meta_extra(self): + def meta_extra(self) -> Optional[Dict]: """Get host extra meta information.""" return self._meta_extra - async def delete(self): + async def delete(self) -> bool: """Issue host deletion via associated provider.""" await self.provider.delete_host(self.host_id, self.name) self._status = STATUS_DELETED From 48fc28157e306c642a77389e0b9dc4c0dcc0eac8 Mon Sep 17 00:00:00 2001 From: Petr Vobornik Date: Fri, 17 Feb 2023 21:28:52 +0100 Subject: [PATCH 3/4] refactor: add MrackSession to FileDBDriver Signed-off-by: Petr Vobornik --- src/mrack/dbdrivers/file.py | 66 ++++++++++++++++--------------------- src/mrack/session.py | 2 +- 2 files changed, 29 insertions(+), 39 deletions(-) diff --git a/src/mrack/dbdrivers/file.py b/src/mrack/dbdrivers/file.py index 7cb49d93..6f7279fa 100644 --- a/src/mrack/dbdrivers/file.py +++ b/src/mrack/dbdrivers/file.py @@ -15,74 +15,64 @@ """File database driver module.""" from os import path +from typing import Dict, Optional +from mrack.domain import domain_from_json from mrack.host import host_from_json +from mrack.session import MrackSession from mrack.utils import load_json, save_to_json +DOMAINS_KEY = "domains" HOSTS_KEY = "hosts" class FileDBDriver: """File database driver. - Serialize and load information into JSON file. + Serialize session's hosts and domains into JSON file. + + Or loads it from JSON file into session. """ - def __init__(self, file_path): + _session: MrackSession + _path: str + _raw_data: Optional[Dict] + + def __init__(self, session: MrackSession, file_path: str): """Initialize DB driver.""" + self._session = session self._path = file_path - self._hosts = {} self._raw_data = None - self.save_on_change = True self.load() def load(self): """Load configuration from file.""" self._hosts = {} if not path.exists(self._path): - self._raw_data = {HOSTS_KEY: {}} - return self._hosts + self._raw_data = { + DOMAINS_KEY: [], + HOSTS_KEY: [], + } + return self._raw_data = load_json(self._path) + raw_hosts = self._raw_data.get(HOSTS_KEY, []) + raw_domains = self._raw_data.get(DOMAINS_KEY, []) self._hosts = {} for raw_host in raw_hosts: - host = host_from_json(raw_host) - self._hosts[host.name] = host + host = host_from_json(self._session, raw_host) + self._session.hosts[host.name] = host - return self._hosts + for raw_domain in raw_domains: + domain = domain_from_json(self._session, raw_domain) + self._session.domains[domain.name] = domain def save(self): """Save configuration to file.""" - hosts = [host.to_json() for host in self._hosts.values()] + hosts = [host.to_json() for host in self._session.hosts.values()] + domains = [domain.to_json() for domain in self._session.domains.values()] self._raw_data[HOSTS_KEY] = hosts + self._raw_data[DOMAINS_KEY] = domains save_to_json(self._path, self._raw_data) - - @property - def hosts(self): - """Get all host objects loaded or to be saved.""" - return self._hosts - - def add_hosts(self, hosts): - """Add a host object. - - Save it to file automatically if `save_on_change` is set to True. - """ - for host in hosts: - self.hosts[host.name] = host - - if self.save_on_change: - self.save() - - def update_hosts(self, hosts): - """Update managed host objects. - - Only adds. - """ - self.add_hosts(hosts) - - def delete_host(self, host): - """Delete host object.""" - if host.name in self.hosts: - del self.hosts[host.name] diff --git a/src/mrack/session.py b/src/mrack/session.py index 5066aef0..9f237816 100644 --- a/src/mrack/session.py +++ b/src/mrack/session.py @@ -93,7 +93,7 @@ def init( def _init_db(self, path: str): """Initialize file database.""" - self._database = FileDBDriver(path) + self._database = FileDBDriver(self, path) @NoSuchFileHandler(error="Provisioning config file not found: {path}") def _init_prov_config(self, path: str): From c3f74da1a4d0ad72cd128b6323541851c08a5e23 Mon Sep 17 00:00:00 2001 From: Petr Vobornik Date: Wed, 22 Feb 2023 17:30:24 +0100 Subject: [PATCH 4/4] refactor: add transformers and providers to session Let's allow sesion to have various transformers and providers. Signed-off-by: Petr Vobornik --- src/mrack/session.py | 16 +++++ src/mrack/transformers/__init__.py | 97 ++++++++++++++++-------------- 2 files changed, 69 insertions(+), 44 deletions(-) diff --git a/src/mrack/session.py b/src/mrack/session.py index 9f237816..06ce83e2 100644 --- a/src/mrack/session.py +++ b/src/mrack/session.py @@ -22,6 +22,8 @@ from mrack.domain import Domain from mrack.errors import ConfigError from mrack.host import Host +from mrack.providers import Registry as ProviderRegistry +from mrack.transformers import Registry as TransformerRegistry from mrack.utils import NoSuchFileHandler, load_yaml @@ -34,12 +36,16 @@ class MrackSession: _metadata: Dict _hosts: Dict[str, Host] _domains: Dict[str, Domain] + _providers: ProviderRegistry + _transformers: TransformerRegistry def __init__(self) -> None: """Init session.""" self._metadata = {} self._hosts = {} self._domains = {} + self._providers = ProviderRegistry() + self._transformers = TransformerRegistry() @property def database(self) -> FileDBDriver: # pylint: disable=invalid-name @@ -71,6 +77,16 @@ def domains(self) -> Dict[str, Domain]: # pylint: disable=invalid-name """Get dictionary of domains where key is domain name.""" return self._domains + @property + def providers(self) -> ProviderRegistry: + """Get ProviderRegistry.""" + return self._providers + + @property + def transformers(self) -> TransformerRegistry: + """Get ProviderRegistry.""" + return self._transformers + def init( self, mrack_config_path: str, diff --git a/src/mrack/transformers/__init__.py b/src/mrack/transformers/__init__.py index 5751fe4c..df5f30c8 100644 --- a/src/mrack/transformers/__init__.py +++ b/src/mrack/transformers/__init__.py @@ -26,57 +26,63 @@ NAME = 0 CLASS = 1 -installed_transformers: Set[Tuple[str, Type[Transformer]]] = set() -IMPORT_ERR_TEMPLATE = "Transformer '%s' not installed, skipping registration" + logger = logging.getLogger(__name__) -try: - from mrack.transformers.aws import CONFIG_KEY as AWS_KEY - from mrack.transformers.aws import AWSTransformer - installed_transformers.add((AWS_KEY, AWSTransformer)) -except ModuleNotFoundError as import_err: - logger.debug(IMPORT_ERR_TEMPLATE, import_err.name) +def discover_transformers() -> Set[Tuple[str, Type[Transformer]]]: + """Discover installed transformers.""" + installed_transformers: Set[Tuple[str, Type[Transformer]]] = set() + IMPORT_ERR_TEMPLATE = "Transformer '%s' not installed, skipping registration" + + try: + from mrack.transformers.aws import CONFIG_KEY as AWS_KEY + from mrack.transformers.aws import AWSTransformer + + installed_transformers.add((AWS_KEY, AWSTransformer)) + except ModuleNotFoundError as import_err: + logger.debug(IMPORT_ERR_TEMPLATE, import_err.name) -try: - from mrack.transformers.beaker import CONFIG_KEY as BEAKER_KEY - from mrack.transformers.beaker import BeakerTransformer + try: + from mrack.transformers.beaker import CONFIG_KEY as BEAKER_KEY + from mrack.transformers.beaker import BeakerTransformer - installed_transformers.add((BEAKER_KEY, BeakerTransformer)) -except ModuleNotFoundError as import_err: - logger.debug(IMPORT_ERR_TEMPLATE, import_err.name) + installed_transformers.add((BEAKER_KEY, BeakerTransformer)) + except ModuleNotFoundError as import_err: + logger.debug(IMPORT_ERR_TEMPLATE, import_err.name) -try: - from mrack.transformers.openstack import CONFIG_KEY as OPENSTACK_KEY - from mrack.transformers.openstack import OpenStackTransformer + try: + from mrack.transformers.openstack import CONFIG_KEY as OPENSTACK_KEY + from mrack.transformers.openstack import OpenStackTransformer - installed_transformers.add((OPENSTACK_KEY, OpenStackTransformer)) -except ModuleNotFoundError as import_err: - logger.debug(IMPORT_ERR_TEMPLATE, import_err.name) + installed_transformers.add((OPENSTACK_KEY, OpenStackTransformer)) + except ModuleNotFoundError as import_err: + logger.debug(IMPORT_ERR_TEMPLATE, import_err.name) -try: - from mrack.transformers.podman import CONFIG_KEY as PODMAN_KEY - from mrack.transformers.podman import PodmanTransformer + try: + from mrack.transformers.podman import CONFIG_KEY as PODMAN_KEY + from mrack.transformers.podman import PodmanTransformer - installed_transformers.add((PODMAN_KEY, PodmanTransformer)) -except ModuleNotFoundError as import_err: - logger.debug(IMPORT_ERR_TEMPLATE, import_err.name) + installed_transformers.add((PODMAN_KEY, PodmanTransformer)) + except ModuleNotFoundError as import_err: + logger.debug(IMPORT_ERR_TEMPLATE, import_err.name) -try: - from mrack.transformers.static import CONFIG_KEY as STATIC_KEY - from mrack.transformers.static import StaticTransformer + try: + from mrack.transformers.static import CONFIG_KEY as STATIC_KEY + from mrack.transformers.static import StaticTransformer - installed_transformers.add((STATIC_KEY, StaticTransformer)) -except ModuleNotFoundError as import_err: - logger.debug(IMPORT_ERR_TEMPLATE, import_err.name) + installed_transformers.add((STATIC_KEY, StaticTransformer)) + except ModuleNotFoundError as import_err: + logger.debug(IMPORT_ERR_TEMPLATE, import_err.name) -try: - from mrack.transformers.virt import CONFIG_KEY as VIRT_KEY - from mrack.transformers.virt import VirtTransformer + try: + from mrack.transformers.virt import CONFIG_KEY as VIRT_KEY + from mrack.transformers.virt import VirtTransformer - installed_transformers.add((VIRT_KEY, VirtTransformer)) -except ModuleNotFoundError as import_err: - logger.debug(IMPORT_ERR_TEMPLATE, import_err.name) + installed_transformers.add((VIRT_KEY, VirtTransformer)) + except ModuleNotFoundError as import_err: + logger.debug(IMPORT_ERR_TEMPLATE, import_err.name) + return installed_transformers class Registry: @@ -110,11 +116,14 @@ def names(self): """Get all registered transformer names.""" return self._transformer_cls.keys() + def discover(self): + """Discover and add installed transformers.""" + installed_transformers = discover_transformers() + for installed_transformer in installed_transformers: + self.register( + installed_transformer[NAME], + installed_transformer[CLASS], + ) -transformers = Registry() -for installed_transformer in installed_transformers: - transformers.register( - installed_transformer[NAME], - installed_transformer[CLASS], - ) +transformers = Registry()