diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e44b7b079..f7fb200d8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,13 +1,13 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v5.0.0 + rev: v6.0.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer - id: check-yaml - id: check-added-large-files - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.11.5 + rev: v0.16.5 hooks: - id: ruff args: [ --fix ] @@ -16,7 +16,7 @@ repos: types_or: ["python"] - repo: https://github.com/pre-commit/mirrors-mypy - rev: 'v1.15.0' + rev: 'v2.3.1' hooks: - id: mypy additional_dependencies: ["types-tabulate", "types-requests"] diff --git a/amlb/__init__.py b/amlb/__init__.py index 1d25ee2ea..33ba8749f 100644 --- a/amlb/__init__.py +++ b/amlb/__init__.py @@ -2,23 +2,23 @@ amlb entrypoint package. """ -from .logger import app_logger as log +from .__version__ import __version__ +from .benchmark import Benchmark, SetupMode from .errors import AutoMLError +from .logger import app_logger as log from .resources import Resources -from .benchmark import Benchmark, SetupMode -from .runners import AWSBenchmark, DockerBenchmark, SingularityBenchmark from .results import TaskResult -from .__version__ import __version__ +from .runners import AWSBenchmark, DockerBenchmark, SingularityBenchmark __all__ = [ - "log", + "AWSBenchmark", "AutoMLError", - "Resources", "Benchmark", "DockerBenchmark", - "SingularityBenchmark", - "AWSBenchmark", + "Resources", "SetupMode", + "SingularityBenchmark", "TaskResult", "__version__", + "log", ] diff --git a/amlb/benchmark.py b/amlb/benchmark.py index 6dc326442..547f04145 100644 --- a/amlb/benchmark.py +++ b/amlb/benchmark.py @@ -10,28 +10,32 @@ from __future__ import annotations -from copy import copy -from enum import Enum -from functools import cached_property -from importlib import import_module, invalidate_caches import logging import math import os import re import signal import sys +from copy import copy +from enum import Enum +from functools import cached_property +from importlib import import_module, invalidate_caches import pandas as pd -from .frameworks.definitions import load_framework_definition -from .job import Job, JobError, SimpleJobRunner, MultiThreadingJobRunner -from .datasets import DataLoader, DataSourceType from .data import DatasetType +from .datasets import DataLoader, DataSourceType from .datautils import read_csv -from .resources import get as rget, config as rconfig, output_dirs as routput_dirs +from .frameworks.definitions import load_framework_definition +from .job import Job, JobError, MultiThreadingJobRunner, SimpleJobRunner +from .resources import config as rconfig +from .resources import get as rget +from .resources import output_dirs as routput_dirs from .results import ErrorResult, Scoreboard, TaskResult from .utils import ( Namespace as ns, +) +from .utils import ( OSMonitoring, as_list, datetime_iso, @@ -51,7 +55,6 @@ touch, ) - log = logging.getLogger(__name__) _setup_dir_ = ".setup" @@ -187,7 +190,7 @@ def setup(self, mode: SetupMode): if mode == SetupMode.skip or mode == SetupMode.auto and self._is_setup_done(): return - log.info("Setting up framework {}.".format(self.framework_name)) + log.info(f"Setting up framework {self.framework_name}.") self._write_setup_env( self.framework_module.__path__[0], **dict(self.framework_def.setup_env) @@ -235,9 +238,7 @@ def resolve_venv(cmd): ) invalidate_caches() - log.info( - "Setup of framework {} completed successfully.".format(self.framework_name) - ) + log.info(f"Setup of framework {self.framework_name} completed successfully.") self._mark_setup_done() @@ -338,15 +339,17 @@ def on_interrupt(*_): # threading.Thread(target=self.cleanup) try: - with signal_handler(signal.SIGINT, on_interrupt): - with OSMonitoring( + with ( + signal_handler(signal.SIGINT, on_interrupt), + OSMonitoring( name=jobs[0].name if len(jobs) == 1 else None, interval_seconds=rconfig().monitoring.interval_seconds, check_on_exit=True, statistics=rconfig().monitoring.statistics, verbosity=rconfig().monitoring.verbosity, - ): - self.job_runner.start() + ), + ): + self.job_runner.start() except (KeyboardInterrupt, InterruptedError): pass finally: @@ -381,7 +384,7 @@ def _get_task_def(self, task_name, include_disabled=False, fail_on_missing=True) ) except StopIteration: if fail_on_missing: - raise ValueError("Incorrect task name: {}.".format(task_name)) + raise ValueError(f"Incorrect task name: {task_name}.") return None if not include_disabled and not Benchmark._is_task_enabled(task_def): raise ValueError( @@ -563,7 +566,7 @@ def __init__( self.git_info = git_info self.measure_inference_time = measure_inference_time self.ext = ns() # used if frameworks require extra config points - self.quantile_levels = list(sorted(quantile_levels)) + self.quantile_levels = sorted(quantile_levels) def __setattr__(self, name, value): if name == "metrics": diff --git a/amlb/benchmarks/file.py b/amlb/benchmarks/file.py index 034af74cc..64f696a50 100644 --- a/amlb/benchmarks/file.py +++ b/amlb/benchmarks/file.py @@ -1,8 +1,8 @@ import logging import os -from typing import List, Tuple, Optional +from typing import List, Optional, Tuple -from amlb.utils import config_load, Namespace +from amlb.utils import Namespace, config_load log = logging.getLogger(__name__) diff --git a/amlb/benchmarks/openml.py b/amlb/benchmarks/openml.py index 98fd6fa30..a359b5b7f 100644 --- a/amlb/benchmarks/openml.py +++ b/amlb/benchmarks/openml.py @@ -6,11 +6,10 @@ import openml import pandas as pd -from openml import OpenMLTask, OpenMLDataset +from openml import OpenMLDataset, OpenMLTask from amlb.utils import Namespace, str_sanitize - log = logging.getLogger(__name__) @@ -67,7 +66,7 @@ def load_openml_tasks_from_suite(domain: str, oml_id: int) -> list[Namespace]: name=str_sanitize(datasets.loc[did]["name"]), description=f"{domain}/d/{did}", openml_task_id=tid, - id="{}.org/t/{}".format(domain, tid), + id=f"{domain}.org/t/{tid}", ) ) return tasks @@ -81,7 +80,7 @@ def load_openml_task_as_definition(domain: str, oml_id: int) -> list[Namespace]: name=str_sanitize(data.name), description=data.description, openml_task_id=task.id, - id="{}.org/t/{}".format(domain, task.id), + id=f"{domain}.org/t/{task.id}", ) ] diff --git a/amlb/benchmarks/parser.py b/amlb/benchmarks/parser.py index 239742831..7d093db61 100644 --- a/amlb/benchmarks/parser.py +++ b/amlb/benchmarks/parser.py @@ -2,9 +2,10 @@ from typing import List, Tuple -from .openml import is_openml_benchmark, load_oml_benchmark +from amlb.utils import Namespace, str_sanitize + from .file import load_file_benchmark -from amlb.utils import str_sanitize, Namespace +from .openml import is_openml_benchmark, load_oml_benchmark def benchmark_load( diff --git a/amlb/data.py b/amlb/data.py index 9a41c6ef8..6e11a967c 100644 --- a/amlb/data.py +++ b/amlb/data.py @@ -13,11 +13,12 @@ from __future__ import annotations +import logging from abc import ABC, abstractmethod +from collections.abc import Iterable from enum import Enum from functools import cached_property -import logging -from typing import List, Union, Iterable +from typing import List, Union import numpy as np import pandas as pd @@ -126,7 +127,6 @@ def data_path(self, format: str) -> str: :param format: the format requested for the data file. Currently supported formats are 'arff', 'csv'. :return: the path to the data-split file in the requested format. """ - pass @cached_property @abstractmethod @@ -134,7 +134,6 @@ def data(self) -> DF: """ :return: all the columns (predictors + target) as a pandas DataFrame. """ - pass @cached_property @profile(logger=log) @@ -199,7 +198,6 @@ def type(self) -> DatasetType: """ :return: the problem type for the current dataset. """ - pass @property @abstractmethod @@ -207,7 +205,6 @@ def train(self) -> Datasplit: """ :return: the data subset used to train the model. """ - pass @property @abstractmethod @@ -215,7 +212,6 @@ def test(self) -> Datasplit: """ :return: the data subset used to score the model. """ - pass @property @abstractmethod @@ -223,7 +219,6 @@ def features(self) -> List[Feature]: """ :return: the list of all features available in the current dataset, target included. """ - pass @property def predictors(self) -> List[Feature]: @@ -238,7 +233,6 @@ def target(self) -> Feature: """ :return: the target feature for the current dataset. """ - pass @profile(logger=log) def release(self) -> None: diff --git a/amlb/datasets/file.py b/amlb/datasets/file.py index f67bd0f99..43de20e5c 100644 --- a/amlb/datasets/file.py +++ b/amlb/datasets/file.py @@ -1,10 +1,10 @@ from __future__ import annotations -from abc import abstractmethod import logging import os import re import tempfile +from abc import abstractmethod from functools import cache, cached_property from typing import List @@ -18,6 +18,8 @@ from ..resources import config as rconfig from ..utils import ( Namespace as ns, +) +from ..utils import ( as_list, list_all_files, path_from_split, @@ -25,8 +27,7 @@ repr_def, split_path, ) - -from .fileutils import is_archive, is_valid_url, unarchive_file, get_file_handler +from .fileutils import get_file_handler, is_archive, is_valid_url, unarchive_file log = logging.getLogger(__name__) diff --git a/amlb/datasets/fileutils.py b/amlb/datasets/fileutils.py index 31d33c70e..d8dcfdf39 100644 --- a/amlb/datasets/fileutils.py +++ b/amlb/datasets/fileutils.py @@ -2,12 +2,13 @@ import os import shutil import tarfile -import boto3 -from botocore.errorfactory import ClientError +import zipfile from urllib.error import URLError from urllib.parse import urlparse from urllib.request import Request, urlopen -import zipfile + +import boto3 +from botocore.errorfactory import ClientError from ..utils import touch diff --git a/amlb/datasets/openml.py b/amlb/datasets/openml.py index 7123cf5ef..8ab77450d 100644 --- a/amlb/datasets/openml.py +++ b/amlb/datasets/openml.py @@ -5,27 +5,29 @@ from __future__ import annotations -import pathlib -from abc import abstractmethod import copy import functools -from functools import cached_property import logging import os +import pathlib import re -from typing import Generic, Tuple, TypeVar, Hashable +from abc import abstractmethod +from collections.abc import Hashable +from functools import cached_property +from typing import Generic, Tuple, TypeVar import arff import numpy as np +import openml as oml import pandas as pd import pandas.api.types as pat -import openml as oml import xmltodict from ..benchmarks.openml import load_openml_task_and_data from ..data import AM, DF, Dataset, DatasetType, Datasplit, Feature from ..datautils import impute_array -from ..resources import config as rconfig, get as rget +from ..resources import config as rconfig +from ..resources import get as rget from ..utils import ( as_list, path_from_split, @@ -34,7 +36,6 @@ unsparsify, ) - # https://github.com/openml/automlbenchmark/pull/574#issuecomment-1646179921 try: set_openml_cache = oml.config.set_cache_directory @@ -68,15 +69,13 @@ def load(self, task_id=None, dataset_id=None, fold=0): if task_id is not None: if dataset_id is not None: log.warning( - "Ignoring dataset id {} as a task id {} was already provided.".format( - dataset_id, task_id - ) + f"Ignoring dataset id {dataset_id} as a task id {task_id} was already provided." ) task, dataset = load_openml_task_and_data(task_id, with_data=True) _, nfolds, _ = task.get_split_dimensions() if fold >= nfolds: raise ValueError( - "OpenML task {} only accepts `fold` < {}.".format(task_id, nfolds) + f"OpenML task {task_id} only accepts `fold` < {nfolds}." ) elif dataset_id is not None: raise NotImplementedError( @@ -420,9 +419,7 @@ def split(self) -> Tuple[str, str]: if not os.path.isfile(train_path) or not os.path.isfile(test_path): X = self.ds._load_full_data("dataframe") train, test = X.iloc[self.train_ind, :], X.iloc[self.test_ind, :] - name_template = "{name}_{{split}}_{fold}".format( - name=self.ds._oml_dataset.name, fold=self.ds.fold - ) + name_template = f"{self.ds._oml_dataset.name}_{{split}}_{self.ds.fold}" self._save_split(train, train_path, name_template.format(split="train")) self._save_split(test, test_path, name_template.format(split="test")) return train_path, test_path diff --git a/amlb/datautils.py b/amlb/datautils.py index 0cda7d479..cbb48d8c8 100644 --- a/amlb/datautils.py +++ b/amlb/datautils.py @@ -12,7 +12,8 @@ import logging import os -from typing import Iterable, Type, Literal, Any, Callable, Tuple, cast, Union +from collections.abc import Iterable +from typing import Any, Callable, Literal, Tuple, Type, Union, cast try: from typing_extensions import TypeAlias @@ -41,14 +42,13 @@ roc_auc_score, ) from sklearn.preprocessing import ( - LabelEncoder, LabelBinarizer, + LabelEncoder, OneHotEncoder, OrdinalEncoder, ) -from .utils import profile, path_from_split, repr_def, split_path, touch - +from .utils import path_from_split, profile, repr_def, split_path, touch log = logging.getLogger(__name__) @@ -281,7 +281,7 @@ def _encode_missing(self) -> bool: def _reshape(self, vec: np.ndarray) -> np.ndarray: return vec if self.for_target else vec.reshape(-1, 1) - def fit(self, vector: Iterable[str] | None) -> "Encoder": + def fit(self, vector: Iterable[str] | None) -> Encoder: """ :param vector: must be a line vector (array) :return: diff --git a/amlb/frameworks/definitions.py b/amlb/frameworks/definitions.py index 2c2c68bc7..dbf3d8c6c 100644 --- a/amlb/frameworks/definitions.py +++ b/amlb/frameworks/definitions.py @@ -5,7 +5,7 @@ import logging import os from dataclasses import dataclass, field -from typing import List, Optional, Union, TYPE_CHECKING +from typing import TYPE_CHECKING, List, Optional, Union from amlb.utils import Namespace, config_load, str_sanitize @@ -274,7 +274,7 @@ def __post_init__(self): def load_framework_definition( - framework_name: str, configuration: "Resources" + framework_name: str, configuration: Resources ) -> Framework: tag = None if ":" in framework_name: diff --git a/amlb/job.py b/amlb/job.py index 1b9922d26..aec5e41e3 100644 --- a/amlb/job.py +++ b/amlb/job.py @@ -10,22 +10,22 @@ from __future__ import annotations import concurrent.futures -from concurrent.futures import ThreadPoolExecutor -from enum import Enum, auto import logging import platform import pprint import queue import signal import threading +from concurrent.futures import ThreadPoolExecutor +from enum import Enum, auto from functools import partial from typing import Callable, List, Optional from .utils import ( + InterruptTimeout, Namespace, - Timer, ThreadSafeCounter, - InterruptTimeout, + Timer, is_main_thread, raise_in_thread, ) @@ -125,14 +125,16 @@ def start(self): interruption_sequence.append(dict(sig=signal.SIGQUIT)) interruption_sequence.append(dict(sig=signal.SIGKILL)) - with Timer() as t: - with InterruptTimeout( + with ( + Timer() as t, + InterruptTimeout( self.timeout, interruptions=interruption_sequence, wait_retry_secs=60, - ): # escalates every minute if the previous interruption was ineffective - if self.set_state(State.running): - result = self._run() + ), + ): # escalates every minute if the previous interruption was ineffective + if self.set_state(State.running): + result = self._run() log.info("Job `%s` executed in %.3f seconds.", self.name, t.duration) log.debug("Job `%s` returned: %s", self.name, result) return Namespace(name=self.name, result=result, duration=t.duration) @@ -195,11 +197,9 @@ def reset(self, state=State.created): def _setup(self): """hook to execute pre-run logic: this is executed in the same thread as the run logic.""" - pass def _run(self): """jobs should implement their run logic in this method.""" - pass def _cancel(self): """hook executed on the job once it's being cancelled by the runner: diff --git a/amlb/resources.py b/amlb/resources.py index 8a37977c4..80780e7bc 100644 --- a/amlb/resources.py +++ b/amlb/resources.py @@ -16,6 +16,9 @@ from amlb.benchmarks.parser import benchmark_load from amlb.frameworks import default_tag, load_framework_definitions + +from .__version__ import __version__ +from .__version__ import _dev_version as dev from .frameworks.definitions import TaskConstraint from .utils import ( Namespace, @@ -25,8 +28,6 @@ touch, ) from .utils.config import TransformRule, config_load, transform_config -from .__version__ import __version__, _dev_version as dev - log = logging.getLogger(__name__) @@ -138,9 +139,7 @@ def framework_definition(self, name, tag=None): tag = default_tag if tag not in self._frameworks: raise ValueError( - "Incorrect tag `{}`: only those among {} are allowed.".format( - tag, self.config.frameworks.tags - ) + f"Incorrect tag `{tag}`: only those among {self.config.frameworks.tags} are allowed." ) frameworks = self._frameworks[tag] log.debug("Available framework definitions:\n%s", frameworks) @@ -180,9 +179,7 @@ def constraint_definition(self, name: str) -> TaskConstraint: constraint = self._constraints[name.lower()] if not constraint: raise ValueError( - "Incorrect constraint definition `{}`: not listed in {}.".format( - name, self.config.benchmarks.constraints_file - ) + f"Incorrect constraint definition `{name}`: not listed in {self.config.benchmarks.constraints_file}." ) return TaskConstraint(**Namespace.dict(constraint)) @@ -243,9 +240,7 @@ def _validate_task(task: Namespace, config_: Namespace, lenient: bool = False): missing.append(conf) if not lenient and len(missing) > 0: raise ValueError( - "{missing} mandatory properties as missing in task definition {taskdef}.".format( - missing=missing, taskdef=task - ) + f"{missing} mandatory properties as missing in task definition {task}." ) for conf in [ @@ -259,9 +254,7 @@ def _validate_task(task: Namespace, config_: Namespace, lenient: bool = False): if task[conf] is None: task[conf] = config_.benchmarks.defaults[conf] log.debug( - "Config `{config}` not set for task {name}, using default `{value}`.".format( - config=conf, name=task.name, value=task[conf] - ) + f"Config `{conf}` not set for task {task.name}, using default `{task[conf]}`." ) if task["metric"] is None: @@ -270,9 +263,9 @@ def _validate_task(task: Namespace, config_: Namespace, lenient: bool = False): conf = "id" if task[conf] is None: task[conf] = ( - "openml.org/t/{}".format(task.openml_task_id) + f"openml.org/t/{task.openml_task_id}" if task["openml_task_id"] is not None - else "openml.org/d/{}".format(task.openml_dataset_id) + else f"openml.org/d/{task.openml_dataset_id}" if task["openml_dataset_id"] is not None else ( ( @@ -291,7 +284,7 @@ def _validate_task(task: Namespace, config_: Namespace, lenient: bool = False): raise ValueError( "task definition must contain an ID or one property " "among ['openml_task_id', 'dataset'] to create an ID, " - "but task definition is {task}".format(task=str(task)) + f"but task definition is {task!s}" ) conf = "ec2_instance_type" @@ -311,18 +304,14 @@ def _validate_task(task: Namespace, config_: Namespace, lenient: bool = False): i_size = i_map.default task[conf] = ".".join([i_series, i_size]) log.debug( - "Config `{config}` not set for task {name}, using default selection `{value}`.".format( - config=conf, name=task.name, value=task[conf] - ) + f"Config `{conf}` not set for task {task.name}, using default selection `{task[conf]}`." ) conf = "ec2_volume_type" if task[conf] is None: task[conf] = config_.aws.ec2.volume_type log.debug( - "Config `{config}` not set for task {name}, using default `{value}`.".format( - config=conf, name=task.name, value=task[conf] - ) + f"Config `{conf}` not set for task {task.name}, using default `{task[conf]}`." ) diff --git a/amlb/results.py b/amlb/results.py index 67f052baf..6287a0ff2 100644 --- a/amlb/results.py +++ b/amlb/results.py @@ -8,19 +8,18 @@ import collections import io import logging -from functools import cache - import math import os import re import statistics -from typing import Union, Any +from functools import cache +from typing import Any, Union import numpy as np -from numpy import nan, sort import pandas as pd import scipy as sci import scipy.sparse +from numpy import nan, sort from typing_extensions import TypeAlias from .data import Dataset, DatasetType, Feature @@ -30,18 +29,20 @@ balanced_accuracy_score, confusion_matrix, fbeta_score, + is_data_frame, log_loss, mean_absolute_error, mean_squared_error, mean_squared_log_error, r2_score, - roc_auc_score, read_csv, - write_csv, - is_data_frame, + roc_auc_score, to_data_frame, + write_csv, ) -from .resources import get as rget, config as rconfig, output_dirs +from .resources import config as rconfig +from .resources import get as rget +from .resources import output_dirs from .utils import ( Namespace, backup_file, @@ -540,9 +541,7 @@ def validate_row(row) -> bool: ) predictions_set = set(preds.unique()) assert predictions_set <= predictors_set, ( - "Predictions column contains unexpected values: {}.".format( - predictions_set - predictors_set - ) + f"Predictions column contains unexpected values: {predictions_set - predictors_set}." ) assert predictions.apply(validate_row, axis=1).all(), ( "Predictions don't always match the predictor with the highest probability." @@ -716,7 +715,7 @@ def evaluate(self, metric): eval_res.value = metric_fn() except Exception as e: log.exception("Failed to compute metric %s: ", metric, e) - eval_res += Namespace(value=nan, message=f"Scoring {metric}: {str(e)}") + eval_res += Namespace(value=nan, message=f"Scoring {metric}: {e!s}") else: pb_type = self.type.name if self.type is not None else "unknown" # raise ValueError(f"Metric {metric} is not supported for {pb_type}.") diff --git a/amlb/runners/__init__.py b/amlb/runners/__init__.py index ecef451da..ff3ff6b11 100644 --- a/amlb/runners/__init__.py +++ b/amlb/runners/__init__.py @@ -7,7 +7,7 @@ from .singularity import SingularityBenchmark __all__ = [ + "AWSBenchmark", "DockerBenchmark", "SingularityBenchmark", - "AWSBenchmark", ] diff --git a/amlb/runners/aws.py b/amlb/runners/aws.py index 4446e01ce..859cf3e01 100644 --- a/amlb/runners/aws.py +++ b/amlb/runners/aws.py @@ -16,21 +16,22 @@ from __future__ import annotations -import datetime -from concurrent.futures import ThreadPoolExecutor import copy as cp +import datetime import datetime as dt -from enum import Enum import itertools import json import logging import math import operator as op import os -from posixpath import join as url_join, relpath as url_relpath import re -import time import threading +import time +from concurrent.futures import ThreadPoolExecutor +from enum import Enum +from posixpath import join as url_join +from posixpath import relpath as url_relpath from typing import cast from urllib.parse import quote_plus as uenc @@ -44,12 +45,15 @@ JobError, MultiThreadingJobRunner, SimpleJobRunner, +) +from ..job import ( State as JobState, ) -from ..resources import config as rconfig, get as rget +from ..resources import config as rconfig +from ..resources import get as rget from ..results import ErrorResult, NoResultError, Scoreboard, TaskResult from ..utils import ( - Namespace as ns, + Namespace, countdown, datetime_iso, file_filter, @@ -62,11 +66,12 @@ str_iter, tail, touch, - Namespace, +) +from ..utils import ( + Namespace as ns, ) from .docker import DockerBenchmark - log = logging.getLogger(__name__) @@ -207,13 +212,11 @@ def _validate(self): def _validate2(self): if self.ami is None: - raise ValueError("Region {} not supported by AMI yet.".format(self.region)) + raise ValueError(f"Region {self.region} not supported by AMI yet.") def setup(self, mode): if mode == SetupMode.skip: - log.warning( - "AWS setup mode set to unsupported {mode}, ignoring.".format(mode=mode) - ) + log.warning(f"AWS setup mode set to unsupported {mode}, ignoring.") # S3 setup to exchange files between local and ec2 instances self.s3 = boto3.resource("s3", region_name=self.region) @@ -470,9 +473,7 @@ def _run(_self): self._forward_params["benchmark_name"] if self.benchmark_path is None or self.benchmark_path.startswith(rconfig().root_dir) - else "{}/{}".format( - resources_root, self._rel_path(self.benchmark_path) - ) + else f"{resources_root}/{self._rel_path(self.benchmark_path)}" ), constraint=self._forward_params["constraint_name"], task_param="" @@ -625,9 +626,7 @@ def log_console(): if inst_desc["abort"]: self._update_instance(job.ext.instance_id, status="aborted") raise AWSError( - "Aborting instance {} for job {}.".format( - job.ext.instance_id, job.name - ) + f"Aborting instance {job.ext.instance_id} for job {job.name}." ) try: state = instance.state["Name"] @@ -883,7 +882,7 @@ def _start_instance( meta_info=None, ) except Exception as e: - fake_iid = "no_instance_{}".format(len(self.instances) + 1) + fake_iid = f"no_instance_{len(self.instances) + 1}" self.instances[fake_iid] = ns( instance=None, key=inst_key, @@ -975,7 +974,7 @@ def _stop_instance(self, instance_id, terminate=None, wait=True): ) except Exception as e: log.warning( - f"Ignoring exception raised while updating instance {instance_id}: {str(e)}" + f"Ignoring exception raised while updating instance {instance_id}: {e!s}" ) def _update_instance(self, instance_id, **kwargs): @@ -1351,9 +1350,7 @@ def _create_instance_profile(self): log.info("Role %s successfully created.", role_name) if iamc.s3_policy_name not in [p.name for p in irole.policies.all()]: - resource_prefix = "arn:aws:s3:::{bucket}*/{root_key}".format( - bucket=bucket_prefix, root_key=str_def(s3c.root_key) - ) # ARN format for s3, cf. https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-arn-format.html + resource_prefix = f"arn:aws:s3:::{bucket_prefix}*/{str_def(s3c.root_key)}" # ARN format for s3, cf. https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-arn-format.html s3_policy_json = json.dumps( { "Version": "2012-10-17", @@ -1361,7 +1358,7 @@ def _create_instance_profile(self): { "Effect": "Allow", "Action": "s3:List*", - "Resource": "arn:aws:s3:::{}*".format(bucket_prefix), + "Resource": f"arn:aws:s3:::{bucket_prefix}*", }, { "Effect": "Allow", diff --git a/amlb/runners/container.py b/amlb/runners/container.py index 94d283b9f..e2f953653 100644 --- a/amlb/runners/container.py +++ b/amlb/runners/container.py @@ -7,18 +7,19 @@ from __future__ import annotations -from abc import abstractmethod import logging import re +from abc import abstractmethod from typing import cast +from ..__version__ import __version__ +from ..__version__ import _dev_version as dev from ..benchmark import Benchmark, SetupMode from ..errors import InvalidStateError from ..frameworks.definitions import Framework from ..job import Job -from ..resources import config as rconfig, get as rget -from ..__version__ import __version__, _dev_version as dev - +from ..resources import config as rconfig +from ..resources import get as rget log = logging.getLogger(__name__) @@ -192,9 +193,7 @@ def _build_image(self, cache=True): if force == "n": raise InvalidStateError( "The image can't be built as the current branch is not clean or up-to-date. " - "Please switch to the expected `{}` branch, and ensure that it is clean before building the container image.".format( - rget().project_info.branch - ) + f"Please switch to the expected `{rget().project_info.branch}` branch, and ensure that it is clean before building the container image." ) create_dev_image = True @@ -210,10 +209,8 @@ def _build_image(self, cache=True): ) if force == "n": raise InvalidStateError( - "The image can't be built as current branch is not tagged as required `{}`. " - "Please switch to the expected tagged branch before building the container image.".format( - expected_branch - ) + f"The image can't be built as current branch is not tagged as required `{expected_branch}`. " + "Please switch to the expected tagged branch before building the container image." ) create_dev_image = True if create_dev_image and not image: diff --git a/amlb/runners/docker.py b/amlb/runners/docker.py index c4eaae445..8ede4c071 100644 --- a/amlb/runners/docker.py +++ b/amlb/runners/docker.py @@ -14,7 +14,6 @@ from ..utils import dir_of, run_cmd, str_digest, str_sanitize, touch from .container import ContainerBenchmark - log = logging.getLogger(__name__) @@ -59,19 +58,9 @@ def _start_container(self, script_params=""): script_extra_params = "--session=" # in combination with `self.output_dirs.session` usage below to prevent creation of 2 sessions locally inst_name = f"{self.sid}.{str_sanitize(str_digest(script_params))}" cmd = ( - "docker run --name {name} {options} {run_as} " - "-v '{input}':/input -v '{output}':/output -v '{custom}':/custom " - "--rm {image} {params} -i /input -o /output -u /custom -s skip -Xrun_mode=docker {extra_params}" - ).format( - name=inst_name, - options=rconfig().docker.run_extra_options, - run_as=run_as, - input=in_dir, - output=self.output_dirs.session, - custom=custom_dir, - image=self.image, - params=script_params, - extra_params=script_extra_params, + f"docker run --name {inst_name} {rconfig().docker.run_extra_options} {run_as} " + f"-v '{in_dir}':/input -v '{self.output_dirs.session}':/output -v '{custom_dir}':/custom " + f"--rm {self.image} {script_params} -i /input -o /output -u /custom -s skip -Xrun_mode=docker {script_extra_params}" ) log.info("Starting docker: %s.", cmd) log.info("Datasets are loaded by default from folder %s.", in_dir) diff --git a/amlb/runners/singularity.py b/amlb/runners/singularity.py index c9b0ada90..7974a7afc 100644 --- a/amlb/runners/singularity.py +++ b/amlb/runners/singularity.py @@ -18,7 +18,6 @@ from ..utils import dir_of, run_cmd, touch from .container import ContainerBenchmark - log = logging.getLogger(__name__) @@ -76,17 +75,9 @@ def _start_container(self, script_params=""): touch(d, as_dir=True) script_extra_params = "--session=" # in combination with `self.output_dirs.session` usage below to prevent creation of 2 sessions locally cmd = ( - "singularity run --pwd /bench {options} " - "-B '{input}':/input -B '{output}':/output -B '{custom}':/custom " - '{image} "{params} -i /input -o /output -u /custom -s skip -Xrun_mode=singularity {extra_params}"' - ).format( - options=rconfig().singularity.run_extra_options, - input=in_dir, - output=self.output_dirs.session, - custom=custom_dir, - image=self.image, - params=script_params, - extra_params=script_extra_params, + f"singularity run --pwd /bench {rconfig().singularity.run_extra_options} " + f"-B '{in_dir}':/input -B '{self.output_dirs.session}':/output -B '{custom_dir}':/custom " + f'{self.image} "{script_params} -i /input -o /output -u /custom -s skip -Xrun_mode=singularity {script_extra_params}"' ) log.info("Starting Singularity: %s.", cmd) log.info("Datasets are loaded by default from folder %s.", in_dir) @@ -112,10 +103,7 @@ def _image_exists(self, image): try: # We pull from docker as there are not yet singularity org accounts run_cmd( - "singularity pull {output_file} docker://{image}".format( - image=self._container_image_name(as_docker_image=True), - output_file=image, - ), + f"singularity pull {image} docker://{self._container_image_name(as_docker_image=True)}", _live_output_=True, ) return True @@ -123,11 +111,7 @@ def _image_exists(self, image): try: # If no docker image, pull from singularity hub run_cmd( - "singularity pull {output_file} library://{library}/{image}".format( - image=self._container_image_name(as_docker_image=True), - output_file=image, - library=rconfig().singularity.library, - ), + f"singularity pull {image} library://{rconfig().singularity.library}/{self._container_image_name(as_docker_image=True)}", _live_output_=True, ) return True diff --git a/amlb/uploads.py b/amlb/uploads.py index 03d310937..32a00e5a3 100644 --- a/amlb/uploads.py +++ b/amlb/uploads.py @@ -1,13 +1,13 @@ import json import logging import pathlib -from collections import OrderedDict import textwrap -from typing import Set, Optional, List +from collections import OrderedDict +from typing import List, Optional, Set import openml import pandas as pd -from openml import OpenMLTask, OpenMLFlow +from openml import OpenMLFlow, OpenMLTask from openml.runs.functions import format_prediction from .utils.core import Namespace diff --git a/amlb/utils/cache.py b/amlb/utils/cache.py index b1deb6257..186788590 100644 --- a/amlb/utils/cache.py +++ b/amlb/utils/cache.py @@ -1,7 +1,9 @@ from __future__ import annotations + import logging +from collections.abc import Sequence from functools import cached_property -from typing import Any, Sequence +from typing import Any log = logging.getLogger(__name__) diff --git a/amlb/utils/config.py b/amlb/utils/config.py index a95ac1b51..2aa624d23 100644 --- a/amlb/utils/config.py +++ b/amlb/utils/config.py @@ -1,9 +1,10 @@ from __future__ import annotations + +import logging +import os from copy import deepcopy from dataclasses import dataclass from importlib.util import find_spec -import logging -import os from typing import Callable, List, Union from .core import Namespace, identity, json_load diff --git a/amlb/utils/core.py b/amlb/utils/core.py index ed1a1dcb4..ae4355c9c 100644 --- a/amlb/utils/core.py +++ b/amlb/utils/core.py @@ -1,15 +1,15 @@ -from ast import literal_eval import base64 -from collections import defaultdict -from collections.abc import Iterable, Sized -from copy import deepcopy -from functools import reduce import hashlib import json import logging import pprint import re import threading +from ast import literal_eval +from collections import defaultdict +from collections.abc import Iterable, Sized +from copy import deepcopy +from functools import reduce log = logging.getLogger(__name__) @@ -240,9 +240,7 @@ def _classname(obj): def repr_def(obj, attributes="public"): - return "{cls}({attrs!r})".format( - cls=_classname(obj), attrs=_attributes(obj, attributes) - ) + return f"{_classname(obj)}({_attributes(obj, attributes)!r})" def noop(*args, **kwargs): @@ -300,13 +298,15 @@ def as_list(*args): def flatten(iterable, flatten_tuple=False, flatten_dict=False): return reduce( lambda left, right: ( - left.extend(right) - if isinstance(right, (list, tuple) if flatten_tuple else list) - else left.extend(right.items()) - if flatten_dict and isinstance(right, dict) - else left.append(right) - ) - or left, + ( + left.extend(right) + if isinstance(right, (list, tuple) if flatten_tuple else list) + else left.extend(right.items()) + if flatten_dict and isinstance(right, dict) + else left.append(right) + ) + or left + ), iterable, [], ) diff --git a/amlb/utils/os.py b/amlb/utils/os.py index 131b559c0..9449637a2 100644 --- a/amlb/utils/os.py +++ b/amlb/utils/os.py @@ -195,8 +195,10 @@ def add_to_archive(file, isdir): walk_apply( path, add_to_archive, - filter_=lambda p: (filter_ is None or filter_(p)) - and not os.path.samefile(dest_archive, p), + filter_=lambda p: ( + (filter_ is None or filter_(p)) + and not os.path.samefile(dest_archive, p) + ), ) diff --git a/amlb/utils/process.py b/amlb/utils/process.py index 1b04eab54..dda1c1ea0 100644 --- a/amlb/utils/process.py +++ b/amlb/utils/process.py @@ -1,9 +1,6 @@ from __future__ import annotations import gc -from concurrent.futures import ThreadPoolExecutor -from contextlib import contextmanager -from functools import partial, wraps import inspect import io import logging @@ -18,12 +15,15 @@ import subprocess import sys import threading -from typing import Dict, List, Union, Tuple, cast +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager +from functools import partial, wraps +from typing import Dict, List, Tuple, Union, cast import psutil from .core import Namespace, as_list, flatten, fn_name -from .os import dir_of, to_mb, path_from_split, split_path +from .os import dir_of, path_from_split, split_path, to_mb from .time import Timeout, Timer log = logging.getLogger(__name__) @@ -416,7 +416,7 @@ def call_target(q, *args, **kwargs): else: return result except queue.Empty: - raise Exception("Subprocess running {} died abruptly.".format(target.__name__)) + raise Exception(f"Subprocess running {target.__name__} died abruptly.") except BaseException: try: kill_proc_tree(p.pid) diff --git a/amlb/utils/serialization.py b/amlb/utils/serialization.py index 5beff1213..5c4d89dc0 100644 --- a/amlb/utils/serialization.py +++ b/amlb/utils/serialization.py @@ -5,7 +5,8 @@ import re from typing import Optional -from .core import Namespace as ns, json_dump, json_load +from .core import Namespace as ns +from .core import json_dump, json_load from .process import profile log = logging.getLogger(__name__) diff --git a/amlb/utils/time.py b/amlb/utils/time.py index 1e23710a2..3d0153193 100644 --- a/amlb/utils/time.py +++ b/amlb/utils/time.py @@ -42,13 +42,13 @@ def datetime_iso( date_sep = time_sep = datetime_sep = micros_sep = "" strf = "" if date: - strf += "%Y{_}%m{_}%d".format(_=date_sep) + strf += f"%Y{date_sep}%m{date_sep}%d" if time: strf += datetime_sep if time: - strf += "%H{_}%M{_}%S".format(_=time_sep) + strf += f"%H{time_sep}%M{time_sep}%S" if micros: - strf += "{_}%f".format(_=micros_sep) + strf += f"{micros_sep}%f" datetime = dt.datetime.utcnow() if datetime is None else datetime return datetime.strftime(strf) diff --git a/docs/website/scripts/generate_index.py b/docs/website/scripts/generate_index.py index 34c7237ea..d29941b09 100644 --- a/docs/website/scripts/generate_index.py +++ b/docs/website/scripts/generate_index.py @@ -1,11 +1,11 @@ from __future__ import annotations +from collections.abc import Iterable, Sequence from pathlib import Path from string import Template +from typing import NamedTuple import tomllib -from typing import NamedTuple, Sequence, Iterable - from generate_navigation import generate_navigation diff --git a/docs/website/scripts/generate_navigation.py b/docs/website/scripts/generate_navigation.py index af9126f12..90b071a44 100644 --- a/docs/website/scripts/generate_navigation.py +++ b/docs/website/scripts/generate_navigation.py @@ -1,8 +1,9 @@ +from collections.abc import Iterable +from pathlib import Path from string import Template +from typing import NamedTuple import tomllib -from pathlib import Path -from typing import NamedTuple, Iterable class NavigationItem(NamedTuple): diff --git a/examples/custom/extensions/Stacking/exec.py b/examples/custom/extensions/Stacking/exec.py index c213185f9..2b5b6ae6b 100644 --- a/examples/custom/extensions/Stacking/exec.py +++ b/examples/custom/extensions/Stacking/exec.py @@ -8,9 +8,14 @@ os.environ["MKL_NUM_THREADS"] = "1" import sklearn -from sklearn.ensemble import GradientBoostingClassifier, GradientBoostingRegressor -from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor -from sklearn.ensemble import StackingClassifier, StackingRegressor +from sklearn.ensemble import ( + GradientBoostingClassifier, + GradientBoostingRegressor, + RandomForestClassifier, + RandomForestRegressor, + StackingClassifier, + StackingRegressor, +) from sklearn.linear_model import ( LinearRegression, LogisticRegression, @@ -45,15 +50,11 @@ def run(dataset, config): } log.info( - "Running Sklearn Stacking Ensemble with a maximum time of {}s on {} cores.".format( - config.max_runtime_seconds, n_jobs - ) + f"Running Sklearn Stacking Ensemble with a maximum time of {config.max_runtime_seconds}s on {n_jobs} cores." ) log.warning("We completely ignore the requirement to stay within the time limit.") log.warning( - "We completely ignore the advice to optimize towards metric: {}.".format( - config.metric - ) + f"We completely ignore the advice to optimize towards metric: {config.metric}." ) if is_classification: diff --git a/frameworks/AutoGluon/__init__.py b/frameworks/AutoGluon/__init__.py index da17c39da..9fb739e67 100644 --- a/frameworks/AutoGluon/__init__.py +++ b/frameworks/AutoGluon/__init__.py @@ -1,7 +1,8 @@ -from amlb.utils import call_script_in_same_dir +from copy import deepcopy + from amlb.benchmark import TaskConfig from amlb.data import Dataset, DatasetType -from copy import deepcopy +from amlb.utils import call_script_in_same_dir def setup(*args, **kwargs): diff --git a/frameworks/AutoGluon/exec.py b/frameworks/AutoGluon/exec.py index f833485ef..ae1867dcb 100644 --- a/frameworks/AutoGluon/exec.py +++ b/frameworks/AutoGluon/exec.py @@ -1,9 +1,9 @@ import logging import os import shutil -import warnings import sys import tempfile +import warnings from typing import Union warnings.simplefilter("ignore") @@ -16,16 +16,16 @@ matplotlib.use("agg") # no need for tk -from autogluon.tabular import TabularPredictor, TabularDataset -from autogluon.core.utils.savers import save_pd, save_pkl, save_json -from autogluon.core.metrics import get_metric, Scorer +from autogluon.core.metrics import Scorer, get_metric +from autogluon.core.utils.savers import save_json, save_pd, save_pkl +from autogluon.tabular import TabularDataset, TabularPredictor from autogluon.tabular.version import __version__ from frameworks.shared.callee import ( call_run, - result, - output_subdir, measure_inference_times, + output_subdir, + result, ) from frameworks.shared.utils import Timer, zip_path diff --git a/frameworks/AutoGluon/exec_ts.py b/frameworks/AutoGluon/exec_ts.py index 71ecfb3d2..3443800c4 100644 --- a/frameworks/AutoGluon/exec_ts.py +++ b/frameworks/AutoGluon/exec_ts.py @@ -1,24 +1,25 @@ import logging -import numpy as np import os -import pandas as pd import shutil import sys import tempfile import warnings +import numpy as np +import pandas as pd + warnings.simplefilter("ignore") if sys.platform == "darwin": os.environ["OMP_NUM_THREADS"] = "1" from autogluon.core.utils.savers import save_pd, save_pkl -from autogluon.timeseries import TimeSeriesPredictor, TimeSeriesDataFrame +from autogluon.timeseries import TimeSeriesDataFrame, TimeSeriesPredictor from autogluon.timeseries.version import __version__ from joblib.externals.loky import get_reusable_executor -from frameworks.shared.callee import call_run, result, output_subdir -from frameworks.shared.utils import Timer, zip_path, load_timeseries_dataset +from frameworks.shared.callee import call_run, output_subdir, result +from frameworks.shared.utils import Timer, load_timeseries_dataset, zip_path log = logging.getLogger(__name__) diff --git a/frameworks/AutoWEKA/exec.py b/frameworks/AutoWEKA/exec.py index 7e3dd18a6..0148cae35 100644 --- a/frameworks/AutoWEKA/exec.py +++ b/frameworks/AutoWEKA/exec.py @@ -6,7 +6,7 @@ from amlb.data import Dataset from amlb.datautils import reorder_dataset from amlb.results import NoResultError, save_predictions -from amlb.utils import dir_of, path_from_split, run_cmd, split_path, Timer +from amlb.utils import Timer, dir_of, path_from_split, run_cmd, split_path log = logging.getLogger(__name__) @@ -24,7 +24,7 @@ def run(dataset: Dataset, config: TaskConfig): metrics_mapping[config.metric] if config.metric in metrics_mapping else None ) if metric is None: - raise ValueError("Performance metric {} not supported.".format(config.metric)) + raise ValueError(f"Performance metric {config.metric} not supported.") train_file = dataset.train.path test_file = dataset.test.path @@ -63,19 +63,17 @@ def run(dataset: Dataset, config: TaskConfig): weka_path=f":{weka_jar}" if os.path.isfile(weka_jar) else "", ) cmd_params = dict( - t='"{}"'.format(train_file), - T='"{}"'.format(test_file), + t=f'"{train_file}"', + T=f'"{test_file}"', memLimit=memLimit, - classifications='"weka.classifiers.evaluation.output.prediction.CSV -distribution -file \\"{}\\""'.format( - weka_file - ), + classifications=f'"weka.classifiers.evaluation.output.prediction.CSV -distribution -file \\"{weka_file}\\""', timeLimit=int(config.max_runtime_seconds / 60), parallelRuns=parallelRuns, metric=metric, seed=config.seed % (1 << 16), # weka accepts only int16 as seeds **training_params, ) - cmd = cmd_root + " ".join(["-{} {}".format(k, v) for k, v in cmd_params.items()]) + cmd = cmd_root + " ".join([f"-{k} {v}" for k, v in cmd_params.items()]) with Timer() as training: run_cmd(cmd, _live_output_=True) diff --git a/frameworks/FEDOT/__init__.py b/frameworks/FEDOT/__init__.py index 49c13b700..77fbc675e 100644 --- a/frameworks/FEDOT/__init__.py +++ b/frameworks/FEDOT/__init__.py @@ -1,7 +1,8 @@ +from copy import deepcopy + from amlb.benchmark import TaskConfig from amlb.data import Dataset, DatasetType from amlb.utils import call_script_in_same_dir -from copy import deepcopy def setup(*args, **kwargs): diff --git a/frameworks/FEDOT/exec.py b/frameworks/FEDOT/exec.py index 0baba5010..e91cf29dc 100644 --- a/frameworks/FEDOT/exec.py +++ b/frameworks/FEDOT/exec.py @@ -4,7 +4,7 @@ from fedot.api.main import Fedot -from frameworks.shared.callee import call_run, result, output_subdir +from frameworks.shared.callee import call_run, output_subdir, result from frameworks.shared.utils import Timer log = logging.getLogger(__name__) diff --git a/frameworks/FEDOT/exec_ts.py b/frameworks/FEDOT/exec_ts.py index e0fb3fc10..30290aae4 100644 --- a/frameworks/FEDOT/exec_ts.py +++ b/frameworks/FEDOT/exec_ts.py @@ -1,16 +1,16 @@ import logging import os from pathlib import Path + import numpy as np import pandas as pd - from fedot.api.main import Fedot -from fedot.core.repository.tasks import Task, TaskTypesEnum, TsForecastingParams from fedot.core.data.data import InputData from fedot.core.repository.dataset_types import DataTypesEnum +from fedot.core.repository.tasks import Task, TaskTypesEnum, TsForecastingParams from fedot.version import __version__ -from frameworks.shared.callee import call_run, result, output_subdir +from frameworks.shared.callee import call_run, output_subdir, result from frameworks.shared.utils import Timer, load_timeseries_dataset log = logging.getLogger(__name__) diff --git a/frameworks/GAMA/exec.py b/frameworks/GAMA/exec.py index a7f5ce10e..3acf542ad 100644 --- a/frameworks/GAMA/exec.py +++ b/frameworks/GAMA/exec.py @@ -15,20 +15,18 @@ import category_encoders -from packaging import version import sklearn - from gama import GamaClassifier, GamaRegressor, __version__ +from packaging import version from frameworks.shared.callee import ( call_run, - result, - output_subdir, measure_inference_times, + output_subdir, + result, ) from frameworks.shared.utils import Timer, touch - log = logging.getLogger(__name__) @@ -54,7 +52,7 @@ def run(dataset, config): metrics_mapping[config.metric] if config.metric in metrics_mapping else None ) if scoring_metric is None: - raise ValueError("Performance metric {} not supported.".format(config.metric)) + raise ValueError(f"Performance metric {config.metric} not supported.") training_params = { k: v for k, v in config.framework_params.items() if not k.startswith("_") diff --git a/frameworks/H2OAutoML/__init__.py b/frameworks/H2OAutoML/__init__.py index 6c9169a46..1b9fd3678 100644 --- a/frameworks/H2OAutoML/__init__.py +++ b/frameworks/H2OAutoML/__init__.py @@ -37,7 +37,7 @@ def docker_commands(*args, setup_cmd=None): {cmd} EXPOSE 54321 EXPOSE 54322 -""".format(cmd="RUN {}".format(setup_cmd) if setup_cmd is not None else "") +""".format(cmd=f"RUN {setup_cmd}" if setup_cmd is not None else "") # There is no network isolation in Singularity, @@ -47,4 +47,4 @@ def docker_commands(*args, setup_cmd=None): def singularity_commands(*args, setup_cmd=None): return """ {cmd} -""".format(cmd="{}".format(setup_cmd) if setup_cmd is not None else "") +""".format(cmd=f"{setup_cmd}" if setup_cmd is not None else "") diff --git a/frameworks/H2OAutoML/exec.py b/frameworks/H2OAutoML/exec.py index 009ed49c7..b7b219460 100644 --- a/frameworks/H2OAutoML/exec.py +++ b/frameworks/H2OAutoML/exec.py @@ -2,31 +2,31 @@ import logging import os import pathlib - -import psutil import re -from packaging import version -import pandas as pd - import h2o +import pandas as pd +import psutil from h2o.automl import H2OAutoML +from packaging import version from frameworks.shared.callee import ( FrameworkError, call_run, + measure_inference_times, output_subdir, result, - measure_inference_times, ) from frameworks.shared.utils import ( Monitoring, - Namespace as ns, Timer, clean_dir, touch, zip_path, ) +from frameworks.shared.utils import ( + Namespace as ns, +) log = logging.getLogger(__name__) @@ -166,9 +166,8 @@ def run(dataset, config): else contextlib.nullcontext() # Py 3.7+ only # else contextlib.contextmanager(lambda: (_ for _ in (0,)))() ) - with Timer() as training: - with monitor: - aml.train(y=dataset.target.index, training_frame=train) + with Timer() as training, monitor: + aml.train(y=dataset.target.index, training_frame=train) log.info(f"Finished fit in {training.duration}s.") if not aml.leader: @@ -283,8 +282,10 @@ def save_artifacts(automl, dataset, config): models_artifacts.append(models_archive) clean_dir( models_dir, - filter_=lambda p: p not in models_artifacts - and os.path.splitext(p)[1] in [".json", ".zip", ""], + filter_=lambda p: ( + p not in models_artifacts + and os.path.splitext(p)[1] in [".json", ".zip", ""] + ), ) if "model_predictions" in artifacts: diff --git a/frameworks/MLNet/exec.py b/frameworks/MLNet/exec.py index 310cc073f..769c0a380 100644 --- a/frameworks/MLNet/exec.py +++ b/frameworks/MLNet/exec.py @@ -1,6 +1,6 @@ # import standard_lib -import logging import json +import logging import os import shutil import tempfile @@ -12,7 +12,7 @@ from amlb.benchmark import TaskConfig from amlb.data import Dataset from amlb.results import NoResultError, save_predictions -from amlb.utils import clean_dir, run_cmd, zip_path, Timer +from amlb.utils import Timer, clean_dir, run_cmd, zip_path from frameworks.shared.callee import output_subdir log = logging.getLogger(__name__) @@ -78,7 +78,7 @@ def run(dataset: Dataset, config: TaskConfig): run_cmd(cmd) log.info(f"Finished fit in {training.duration}s.") - train_result_json = os.path.join(output_dir, "{}.mbconfig".format(config.fold)) + train_result_json = os.path.join(output_dir, f"{config.fold}.mbconfig") if not os.path.exists(train_result_json): raise NoResultError("MLNet failed producing any prediction.") diff --git a/frameworks/MLPlan/exec.py b/frameworks/MLPlan/exec.py index 6e58fe599..3e9583804 100644 --- a/frameworks/MLPlan/exec.py +++ b/frameworks/MLPlan/exec.py @@ -1,20 +1,18 @@ import glob +import json import logging import os -import json import re import tempfile -from frameworks.shared.callee import call_run, result, output_subdir +from frameworks.shared.callee import call_run, output_subdir, result from frameworks.shared.utils import Timer, run_cmd log = logging.getLogger(__name__) def run(dataset, config): - jar_file = glob.glob( - "{here}/lib/mlplan/mlplan-cli*.jar".format(here=os.path.dirname(__file__)) - )[0] + jar_file = glob.glob(f"{os.path.dirname(__file__)}/lib/mlplan/mlplan-cli*.jar")[0] version = re.match(r".*/mlplan-cli-(.*).jar", jar_file)[1] log.info(f"\n**** ML-Plan [v{version}] ****\n") @@ -37,9 +35,7 @@ def run(dataset, config): metrics_mapping[config.metric] if config.metric in metrics_mapping else None ) if metric is None: - raise ValueError( - "Performance metric {} is not supported.".format(config.metric) - ) + raise ValueError(f"Performance metric {config.metric} is not supported.") train_file = dataset.train.path test_file = dataset.test.path @@ -79,8 +75,8 @@ def run(dataset, config): with tempfile.TemporaryDirectory() as tmp_dir: cmd_params = dict( - f='"{}"'.format(train_file), - p='"{}"'.format(test_file), + f=f'"{train_file}"', + p=f'"{test_file}"', t=config.max_runtime_seconds, ncpus=config.cores, l=metric, @@ -92,9 +88,7 @@ def run(dataset, config): **training_params, ) - cmd = cmd_root + "".join( - [" -{} {}".format(k, v) for k, v in cmd_params.items()] - ) + cmd = cmd_root + "".join([f" -{k} {v}" for k, v in cmd_params.items()]) with Timer() as training: run_cmd(cmd, _live_output_=True) diff --git a/frameworks/NaiveAutoML/exec.py b/frameworks/NaiveAutoML/exec.py index 67921ff7c..30e497524 100644 --- a/frameworks/NaiveAutoML/exec.py +++ b/frameworks/NaiveAutoML/exec.py @@ -17,21 +17,21 @@ os.environ["OPENBLAS_NUM_THREADS"] = "1" os.environ["MKL_NUM_THREADS"] = "1" +from naiveautoml import NaiveAutoML + from frameworks.shared.callee import ( call_run, - result, - output_subdir, measure_inference_times, + output_subdir, + result, ) from frameworks.shared.utils import Timer -from naiveautoml import NaiveAutoML - log = logging.getLogger(__name__) def run(dataset, config): - pip_list = subprocess.run("python -m pip list".split(), capture_output=True) + pip_list = subprocess.run(["python", "-m", "pip", "list"], capture_output=True) match = re.search( r"naiveautoml\s+([^\n]+)", pip_list.stdout.decode(), flags=re.IGNORECASE ) diff --git a/frameworks/RandomForest/exec.py b/frameworks/RandomForest/exec.py index 4be16ac32..0496f6561 100644 --- a/frameworks/RandomForest/exec.py +++ b/frameworks/RandomForest/exec.py @@ -14,7 +14,7 @@ import sklearn from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor -from frameworks.shared.callee import call_run, result, measure_inference_times +from frameworks.shared.callee import call_run, measure_inference_times, result from frameworks.shared.utils import Timer log = logging.getLogger(os.path.basename(__file__)) @@ -54,14 +54,10 @@ def run(dataset, config): memory_margin = config.framework_params.get("_memory_margin", 0.9) log.info( - "Running RandomForest with a maximum time of {}s on {} cores.".format( - config.max_runtime_seconds, n_jobs - ) + f"Running RandomForest with a maximum time of {config.max_runtime_seconds}s on {n_jobs} cores." ) log.warning( - "We completely ignore the advice to optimize towards metric: {}.".format( - config.metric - ) + f"We completely ignore the advice to optimize towards metric: {config.metric}." ) estimator = RandomForestClassifier if is_classification else RandomForestRegressor diff --git a/frameworks/SapientML/exec.py b/frameworks/SapientML/exec.py index 0a80ba883..f03463336 100644 --- a/frameworks/SapientML/exec.py +++ b/frameworks/SapientML/exec.py @@ -2,11 +2,12 @@ import os import tempfile as tmp -from frameworks.shared.callee import call_run, result -from frameworks.shared.utils import Timer from sapientml import SapientML from sklearn.preprocessing import OneHotEncoder +from frameworks.shared.callee import call_run, result +from frameworks.shared.utils import Timer + os.environ["JOBLIB_TEMP_FOLDER"] = tmp.gettempdir() os.environ["OMP_NUM_THREADS"] = "1" os.environ["OPENBLAS_NUM_THREADS"] = "1" diff --git a/frameworks/TPOT/exec.py b/frameworks/TPOT/exec.py index 4883a6f3f..7bbb66078 100644 --- a/frameworks/TPOT/exec.py +++ b/frameworks/TPOT/exec.py @@ -18,13 +18,12 @@ from frameworks.shared.callee import ( call_run, + measure_inference_times, output_subdir, result, - measure_inference_times, ) from frameworks.shared.utils import Timer, is_sparse - log = logging.getLogger(__name__) @@ -48,7 +47,7 @@ def run(dataset, config): metrics_mapping[config.metric] if config.metric in metrics_mapping else None ) if scoring_metric is None: - raise ValueError("Performance metric {} not supported.".format(config.metric)) + raise ValueError(f"Performance metric {config.metric} not supported.") X_train = dataset.train.X y_train = dataset.train.y diff --git a/frameworks/TunedRandomForest/exec.py b/frameworks/TunedRandomForest/exec.py index 1fbd6e2af..c66cad100 100644 --- a/frameworks/TunedRandomForest/exec.py +++ b/frameworks/TunedRandomForest/exec.py @@ -17,14 +17,14 @@ os.environ["OPENBLAS_NUM_THREADS"] = "1" os.environ["MKL_NUM_THREADS"] = "1" -import psutil import pandas as pd +import psutil import sklearn +from custom_validate import cross_validate from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor -from frameworks.shared.callee import call_run, result, measure_inference_times +from frameworks.shared.callee import call_run, measure_inference_times, result from frameworks.shared.utils import Timer -from custom_validate import cross_validate log = logging.getLogger(__name__) @@ -70,9 +70,7 @@ def run(dataset, config): y_train, y_test = dataset.train.y, dataset.test.y log.info( - "Running RandomForest with a maximum time of {}s on {} cores.".format( - config.max_runtime_seconds, n_jobs - ) + f"Running RandomForest with a maximum time of {config.max_runtime_seconds}s on {n_jobs} cores." ) estimator = RandomForestClassifier if is_classification else RandomForestRegressor diff --git a/frameworks/autosklearn/exec.py b/frameworks/autosklearn/exec.py index d42eb23fb..eb0779b99 100644 --- a/frameworks/autosklearn/exec.py +++ b/frameworks/autosklearn/exec.py @@ -15,16 +15,16 @@ os.environ["OPENBLAS_NUM_THREADS"] = "1" os.environ["MKL_NUM_THREADS"] = "1" import autosklearn +from autosklearn import metrics from autosklearn.estimators import AutoSklearnClassifier, AutoSklearnRegressor from autosklearn.experimental.askl2 import AutoSklearn2Classifier -import autosklearn.metrics as metrics from packaging import version from frameworks.shared.callee import ( call_run, - result, - output_subdir, measure_inference_times, + output_subdir, + result, ) from frameworks.shared.utils import Timer, system_memory_mb, walk_apply, zip_path diff --git a/frameworks/flaml/exec.py b/frameworks/flaml/exec.py index 370f93696..9917149cb 100644 --- a/frameworks/flaml/exec.py +++ b/frameworks/flaml/exec.py @@ -7,9 +7,9 @@ from frameworks.shared.callee import ( call_run, - result, - output_subdir, measure_inference_times, + output_subdir, + result, ) from frameworks.shared.utils import Timer @@ -24,7 +24,7 @@ def run(dataset, config): is_classification = config.type == "classification" time_budget = config.max_runtime_seconds n_jobs = config.framework_params.get("_n_jobs", config.cores) - log.info("Running FLAML with {} number of cores".format(config.cores)) + log.info(f"Running FLAML with {config.cores} number of cores") aml = AutoML() # Mapping of benchmark metrics to flaml metrics diff --git a/frameworks/hyperoptsklearn/exec.py b/frameworks/hyperoptsklearn/exec.py index 48e2217b1..8b6605e28 100644 --- a/frameworks/hyperoptsklearn/exec.py +++ b/frameworks/hyperoptsklearn/exec.py @@ -11,11 +11,11 @@ os.environ["MKL_NUM_THREADS"] = "1" from hpsklearn import HyperoptEstimator, any_classifier, any_regressor from sklearn.metrics import ( - roc_auc_score, f1_score, mean_absolute_error, mean_squared_error, mean_squared_log_error, + roc_auc_score, ) from frameworks.shared.callee import call_run, result @@ -34,8 +34,8 @@ def default(): metrics_to_loss_mapping = dict( acc=(default, False), # lambda y, pred: 1.0 - accuracy_score(y, pred) - auc=(lambda y, pred: 1.0 - roc_auc_score(y, pred), False), # noqa: E731 - f1=(lambda y, pred: 1.0 - f1_score(y, pred), False), # noqa: E731 + auc=(lambda y, pred: 1.0 - roc_auc_score(y, pred), False), + f1=(lambda y, pred: 1.0 - f1_score(y, pred), False), # logloss=(log_loss, True), mae=(mean_absolute_error, False), mse=(mean_squared_error, False), diff --git a/frameworks/lightautoml/exec.py b/frameworks/lightautoml/exec.py index d2ccf0a21..4a592b9b5 100644 --- a/frameworks/lightautoml/exec.py +++ b/frameworks/lightautoml/exec.py @@ -10,15 +10,15 @@ matplotlib.use("agg") # no need for tk -from lightautoml.tasks import Task -from lightautoml.automl.presets.tabular_presets import TabularUtilizedAutoML from lightautoml import __version__ +from lightautoml.automl.presets.tabular_presets import TabularUtilizedAutoML +from lightautoml.tasks import Task from frameworks.shared.callee import ( call_run, - result, - output_subdir, measure_inference_times, + output_subdir, + result, ) from frameworks.shared.utils import Timer diff --git a/frameworks/mljarsupervised/exec.py b/frameworks/mljarsupervised/exec.py index 44003c45c..ad303d2dd 100644 --- a/frameworks/mljarsupervised/exec.py +++ b/frameworks/mljarsupervised/exec.py @@ -1,6 +1,6 @@ +import logging import os import shutil -import logging from typing import Union import matplotlib @@ -13,9 +13,9 @@ from frameworks.shared.callee import ( call_run, - result, - output_subdir, measure_inference_times, + output_subdir, + result, ) from frameworks.shared.utils import Timer diff --git a/frameworks/oboe/exec.py b/frameworks/oboe/exec.py index ebb0232a6..74a53e577 100644 --- a/frameworks/oboe/exec.py +++ b/frameworks/oboe/exec.py @@ -2,12 +2,10 @@ import os import sys -from sklearn.model_selection import StratifiedKFold import numpy as np +from sklearn.model_selection import StratifiedKFold -sys.path.append( - "{}/lib/oboe/automl".format(os.path.realpath(os.path.dirname(__file__))) -) +sys.path.append(f"{os.path.realpath(os.path.dirname(__file__))}/lib/oboe/automl") from oboe import AutoLearner from frameworks.shared.callee import call_run, result @@ -46,7 +44,7 @@ def kfold_fit_validate(self, x_train, y_train, n_folds, random_state=None): self.cv_predictions = y_predicted self.sampled = True if self.verbose: - print("{} {} complete.".format(self.algorithm, self.hyperparameters)) + print(f"{self.algorithm} {self.hyperparameters} complete.") return cv_errors, y_predicted @@ -71,14 +69,10 @@ def run(dataset, config): n_cores = config.framework_params.get("_n_cores", config.cores) log.info( - "Running oboe with a maximum time of {}s on {} cores.".format( - config.max_runtime_seconds, n_cores - ) + f"Running oboe with a maximum time of {config.max_runtime_seconds}s on {n_cores} cores." ) log.warning( - "We completely ignore the advice to optimize towards metric: {}.".format( - config.metric - ) + f"We completely ignore the advice to optimize towards metric: {config.metric}." ) aml = AutoLearner( diff --git a/frameworks/shared/callee.py b/frameworks/shared/callee.py index 23b41620a..89f479b53 100644 --- a/frameworks/shared/callee.py +++ b/frameworks/shared/callee.py @@ -5,19 +5,21 @@ import signal import sys from collections import defaultdict -from typing import Callable, Any, Tuple, TypeVar - +from typing import Any, Callable, Tuple, TypeVar from .utils import ( InterruptTimeout, - Namespace as ns, + Timer, + deserialize_data, json_dump, json_loads, kill_proc_tree, + serialize_data, touch, ) -from .utils import deserialize_data, serialize_data, Timer - +from .utils import ( + Namespace as ns, +) log = logging.getLogger(__name__) diff --git a/frameworks/shared/caller.py b/frameworks/shared/caller.py index 143f69b1d..6f81d50e3 100644 --- a/frameworks/shared/caller.py +++ b/frameworks/shared/caller.py @@ -12,18 +12,22 @@ from amlb.data import Dataset from amlb.resources import config as rconfig from amlb.results import NoResultError, save_predictions -from amlb.utils import json_dump, Namespace +from amlb.utils import Namespace, json_dump from .utils import ( Namespace as ns, +) +from .utils import ( Timer, + deserialize_data, dir_of, - run_cmd, + is_serializable_data, json_dumps, json_load, profile, + run_cmd, + serialize_data, ) -from .utils import is_serializable_data, deserialize_data, serialize_data log = logging.getLogger(__name__) diff --git a/frameworks/shared/utils.py b/frameworks/shared/utils.py index 3ef151e14..07dec1686 100644 --- a/frameworks/shared/utils.py +++ b/frameworks/shared/utils.py @@ -1,9 +1,10 @@ -from importlib import import_module import importlib.util import logging import os -import pandas as pd import sys +from importlib import import_module + +import pandas as pd def setup_logger(): @@ -60,6 +61,6 @@ def load_timeseries_dataset(dataset): utils = load_amlb_module("amlb.utils") # unorthodox for it's only now that we can safely import those functions -from amlb.utils import * # noqa: E402, F403 +from amlb.utils import * __all__ = [s for s in dir() if not s.startswith("_") and s not in __no_export] diff --git a/recover_results.py b/recover_results.py index 62ca00e45..9ccad7fb5 100644 --- a/recover_results.py +++ b/recover_results.py @@ -1,13 +1,12 @@ import argparse import os -# prevent asap other modules from defining the root logger using basicConfig -import amlb.logger - - import amlb -from amlb.utils import Namespace as ns, config_load +# prevent asap other modules from defining the root logger using basicConfig +import amlb.logger +from amlb.utils import Namespace as ns +from amlb.utils import config_load parser = argparse.ArgumentParser() parser.add_argument("instances", type=str, help="The path to an instances.csv file.") diff --git a/runbenchmark.py b/runbenchmark.py index 431e865ef..dba4b604d 100644 --- a/runbenchmark.py +++ b/runbenchmark.py @@ -5,24 +5,26 @@ import shutil import sys -# prevent asap other modules from defining the root logger using basicConfig -import amlb.logger - import openml import amlb + +# prevent asap other modules from defining the root logger using basicConfig +import amlb.logger +from amlb import AutoMLError, log +from amlb.defaults import default_dirs from amlb.utils import ( - Namespace as ns, + Namespace, + StaleProcessError, config_load, datetime_iso, str2bool, str_sanitize, zip_path, - StaleProcessError, - Namespace, ) -from amlb import log, AutoMLError -from amlb.defaults import default_dirs +from amlb.utils import ( + Namespace as ns, +) parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter) parser.add_argument( @@ -181,7 +183,7 @@ "--openml-run-tag", type=str, default=None, - help="Tag that will be saved in metadata and OpenML runs created during upload, must match '([a-zA-Z0-9_\-\.])+'.", + help=r"Tag that will be saved in metadata and OpenML runs created during upload, must match '([a-zA-Z0-9_\-\.])+'.", ) parser.add_argument( @@ -252,12 +254,8 @@ else {} ) | ns(console="INFO", app="DEBUG", root="INFO") # adding defaults if needed amlb.logger.setup( - log_file=os.path.join( - log_dir, "{script}.{now}.log".format(script=script_name, now=now_str) - ), - root_file=os.path.join( - log_dir, "{script}.{now}.full.log".format(script=script_name, now=now_str) - ), + log_file=os.path.join(log_dir, f"{script_name}.{now_str}.log"), + root_file=os.path.join(log_dir, f"{script_name}.{now_str}.full.log"), root_level=log_levels.root, app_level=log_levels.app, console_level=log_levels.console, diff --git a/scripts/find_matching_datasets.py b/scripts/find_matching_datasets.py index 4e75db502..cb56510dc 100644 --- a/scripts/find_matching_datasets.py +++ b/scripts/find_matching_datasets.py @@ -1,10 +1,10 @@ +# I don't have OpenML installed locally +import sys + import arff import requests import yaml -# I don't have OpenML installed locally -import sys - sys.path.append("D:\\repositories/openml-python/") import openml @@ -53,7 +53,7 @@ def try_get_did_for_task(tid): [try_get_did_for_task(tid) for tid in benchmark_tids if tid is not None] ) autosklearn_dids = set( - (try_get_did_for_task(tid) for tid in autosklearn_tids if tid is not None) + try_get_did_for_task(tid) for tid in autosklearn_tids if tid is not None ) print(set(benchmark_dids) & set(autosklearn_dids)) diff --git a/tests/conftest.py b/tests/conftest.py index 3f93aea90..84ff14584 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,7 @@ import os import pytest + from amlb import Resources, resources from amlb.defaults import default_dirs from amlb.utils import Namespace, config_load diff --git a/tests/unit/amlb/benchmarks/test_benchmark.py b/tests/unit/amlb/benchmarks/test_benchmark.py index b088c1700..1b015e7bb 100644 --- a/tests/unit/amlb/benchmarks/test_benchmark.py +++ b/tests/unit/amlb/benchmarks/test_benchmark.py @@ -3,7 +3,7 @@ import pytest -from amlb import Benchmark, SetupMode, resources, DockerBenchmark, SingularityBenchmark +from amlb import Benchmark, DockerBenchmark, SetupMode, SingularityBenchmark, resources from amlb.job import JobError from amlb.utils import Namespace diff --git a/tests/unit/amlb/benchmarks/test_openml.py b/tests/unit/amlb/benchmarks/test_openml.py index ea349ee20..7d4c731b1 100644 --- a/tests/unit/amlb/benchmarks/test_openml.py +++ b/tests/unit/amlb/benchmarks/test_openml.py @@ -3,8 +3,8 @@ from amlb.benchmarks.openml import ( is_openml_benchmark, - load_openml_task_as_definition, load_oml_benchmark, + load_openml_task_as_definition, ) from amlb.utils import Namespace diff --git a/tests/unit/amlb/datasets/file/test_file_dataloader.py b/tests/unit/amlb/datasets/file/test_file_dataloader.py index 78198607d..0c091a175 100644 --- a/tests/unit/amlb/datasets/file/test_file_dataloader.py +++ b/tests/unit/amlb/datasets/file/test_file_dataloader.py @@ -3,13 +3,14 @@ import numpy as np import pandas as pd -import pytest import pandas.api.types as pat +import pytest -from amlb.resources import from_configs from amlb.data import DatasetType from amlb.datasets.file import FileLoader -from amlb.utils import Namespace as ns, path_from_split, split_path +from amlb.resources import from_configs +from amlb.utils import Namespace as ns +from amlb.utils import path_from_split, split_path here = os.path.realpath(os.path.dirname(__file__)) res = os.path.join(here, "resources") diff --git a/tests/unit/amlb/datasets/openml/test_openml_dataloader.py b/tests/unit/amlb/datasets/openml/test_openml_dataloader.py index c3340f9a9..d68b8c898 100644 --- a/tests/unit/amlb/datasets/openml/test_openml_dataloader.py +++ b/tests/unit/amlb/datasets/openml/test_openml_dataloader.py @@ -5,9 +5,9 @@ import pandas as pd import pytest -from amlb.resources import from_configs from amlb.data import DatasetType from amlb.datasets.openml import OpenmlLoader +from amlb.resources import from_configs from amlb.utils import Namespace as ns diff --git a/tests/unit/amlb/frameworks/definitions/test_add_default.py b/tests/unit/amlb/frameworks/definitions/test_add_default.py index 74c22f1de..8cb485885 100644 --- a/tests/unit/amlb/frameworks/definitions/test_add_default.py +++ b/tests/unit/amlb/frameworks/definitions/test_add_default.py @@ -1,12 +1,13 @@ import pytest + from amlb.frameworks.definitions import ( + _add_default_image, _add_default_module, - _add_default_version, + _add_default_params, _add_default_setup_args, - _add_default_setup_script, _add_default_setup_cmd, - _add_default_params, - _add_default_image, + _add_default_setup_script, + _add_default_version, ) from amlb.utils import Namespace diff --git a/tests/unit/amlb/frameworks/definitions/test_framework_definition_processing.py b/tests/unit/amlb/frameworks/definitions/test_framework_definition_processing.py index f2141d61f..99879560d 100644 --- a/tests/unit/amlb/frameworks/definitions/test_framework_definition_processing.py +++ b/tests/unit/amlb/frameworks/definitions/test_framework_definition_processing.py @@ -1,13 +1,14 @@ import pytest -from amlb.utils import Namespace + from amlb.frameworks.definitions import ( - _sanitize_and_add_defaults, _add_framework_name, _find_all_parents, - _update_frameworks_with_parent_definitions, - _remove_self_reference_extensions, _remove_frameworks_with_unknown_parent, + _remove_self_reference_extensions, + _sanitize_and_add_defaults, + _update_frameworks_with_parent_definitions, ) +from amlb.utils import Namespace def test_remove_frameworks_with_unknown_parent_removes_framework_with_unknown_parent(): diff --git a/tests/unit/amlb/frameworks/definitions/test_load_and_merge_framework_definitions.py b/tests/unit/amlb/frameworks/definitions/test_load_and_merge_framework_definitions.py index b8decf0ab..b8e064430 100644 --- a/tests/unit/amlb/frameworks/definitions/test_load_and_merge_framework_definitions.py +++ b/tests/unit/amlb/frameworks/definitions/test_load_and_merge_framework_definitions.py @@ -1,8 +1,10 @@ import os + import pytest + from amlb.frameworks.definitions import ( - default_tag, _load_and_merge_framework_definitions, + default_tag, ) here = os.path.realpath(os.path.dirname(__file__)) diff --git a/tests/unit/amlb/frameworks/definitions/test_load_framework_definitions.py b/tests/unit/amlb/frameworks/definitions/test_load_framework_definitions.py index 27b0fb86b..dba473fc9 100644 --- a/tests/unit/amlb/frameworks/definitions/test_load_framework_definitions.py +++ b/tests/unit/amlb/frameworks/definitions/test_load_framework_definitions.py @@ -1,5 +1,7 @@ import os + import pytest + from amlb.frameworks.definitions import default_tag, load_framework_definitions here = os.path.realpath(os.path.dirname(__file__)) diff --git a/tests/unit/amlb/job/dummy.py b/tests/unit/amlb/job/dummy.py index b29f8da3e..10e9f2a37 100644 --- a/tests/unit/amlb/job/dummy.py +++ b/tests/unit/amlb/job/dummy.py @@ -1,6 +1,7 @@ import time -from amlb.job import Job, State as JobState +from amlb.job import Job +from amlb.job import State as JobState class DummyJob(Job): diff --git a/tests/unit/amlb/job/test_MultiThreadingJobRunner.py b/tests/unit/amlb/job/test_MultiThreadingJobRunner.py index 57debfeff..9b6f4c984 100644 --- a/tests/unit/amlb/job/test_MultiThreadingJobRunner.py +++ b/tests/unit/amlb/job/test_MultiThreadingJobRunner.py @@ -4,12 +4,11 @@ from unittest.mock import patch import pytest +from dummy import DummyJob from amlb.job import MultiThreadingJobRunner, State from amlb.utils import Timeout -from dummy import DummyJob - steps_per_job = 6 diff --git a/tests/unit/amlb/job/test_SimpleJobRunner.py b/tests/unit/amlb/job/test_SimpleJobRunner.py index 64b933dce..30b95d1c0 100644 --- a/tests/unit/amlb/job/test_SimpleJobRunner.py +++ b/tests/unit/amlb/job/test_SimpleJobRunner.py @@ -2,12 +2,11 @@ from unittest.mock import patch import pytest +from dummy import DummyJob from amlb.job import SimpleJobRunner from amlb.utils import Timeout -from dummy import DummyJob - steps_per_job = 6 diff --git a/tests/unit/amlb/uploads/test_file_loading.py b/tests/unit/amlb/uploads/test_file_loading.py index 7c8f36d94..3ee0ac991 100644 --- a/tests/unit/amlb/uploads/test_file_loading.py +++ b/tests/unit/amlb/uploads/test_file_loading.py @@ -1,16 +1,16 @@ -from collections import OrderedDict import pathlib +from collections import OrderedDict import openml import pandas as pd import pytest from amlb.uploads import ( - _load_predictions, - _load_fold, + _extract_and_format_hyperparameter_configuration, _get_flow, + _load_fold, + _load_predictions, _load_task_data, - _extract_and_format_hyperparameter_configuration, _upload_results, ) diff --git a/tests/unit/amlb/utils/process/test_InterruptTimeout.py b/tests/unit/amlb/utils/process/test_InterruptTimeout.py index beeaf5db1..bc75428be 100644 --- a/tests/unit/amlb/utils/process/test_InterruptTimeout.py +++ b/tests/unit/amlb/utils/process/test_InterruptTimeout.py @@ -11,47 +11,46 @@ @pytest.mark.requires_unixlike def test_interruption_behaves_like_a_keyboard_interruption_by_default(): timeout = 1 - with Timer() as t: - with pytest.raises(KeyboardInterrupt): - with InterruptTimeout(timeout_secs=timeout): - for i in range(100): - time.sleep(0.1) + with Timer() as t, pytest.raises(KeyboardInterrupt): + with InterruptTimeout(timeout_secs=timeout): + for i in range(100): + time.sleep(0.1) assert t.duration - timeout < 1 assert i < 11 def test_interruption_with_sig_as_None(): timeout = 1 - with Timer() as t: - with InterruptTimeout(timeout_secs=timeout, sig=None): - for i in range(100): - time.sleep(0.1) + with Timer() as t, InterruptTimeout(timeout_secs=timeout, sig=None): + for i in range(100): + time.sleep(0.1) assert t.duration - timeout < 1 assert i < 11 def test_interruption_with_sig_as_error_class(): timeout = 1 - with Timer() as t: - with pytest.raises( + with ( + Timer() as t, + pytest.raises( TimeoutError, match=r"Interrupting thread (.*) after 1s timeout." - ): - with InterruptTimeout(timeout_secs=timeout, sig=TimeoutError): - for i in range(100): - time.sleep(0.1) + ), + InterruptTimeout(timeout_secs=timeout, sig=TimeoutError), + ): + for i in range(100): + time.sleep(0.1) assert t.duration - timeout < 1 assert i < 11 def test_interruption_with_sig_as_error_instance(): timeout = 1 - with Timer() as t: - with pytest.raises(TimeoutError, match=r"user provided error"): - with InterruptTimeout( - timeout_secs=timeout, sig=TimeoutError("user provided error") - ): - for i in range(100): - time.sleep(0.1) + with Timer() as t, pytest.raises(TimeoutError, match=r"user provided error"): + with InterruptTimeout( + timeout_secs=timeout, sig=TimeoutError("user provided error") + ): + for i in range(100): + time.sleep(0.1) assert t.duration - timeout < 1 assert i < 11 @@ -63,11 +62,10 @@ def _handler(*_): with signal_handler(signal.SIGTERM, _handler): timeout = 1 - with Timer() as t: - with pytest.raises(TimeoutError, match=r"from handler"): - with InterruptTimeout(timeout_secs=timeout, sig=signal.SIGTERM): - for i in range(100): - time.sleep(0.1) + with Timer() as t, pytest.raises(TimeoutError, match=r"from handler"): + with InterruptTimeout(timeout_secs=timeout, sig=signal.SIGTERM): + for i in range(100): + time.sleep(0.1) assert t.duration - timeout < 1 assert i < 11 @@ -92,15 +90,14 @@ def _handler(*_): before = Mock() timeout = 1 - with Timer() as t: - with pytest.raises(TimeoutError, match=r"from handler"): - with InterruptTimeout( - timeout_secs=timeout, - interruptions=[dict(sig=signal.SIGINT), dict(sig=signal.SIGTERM)], - before_interrupt=before, - ): - for i in range(100): - time.sleep(0.1) + with Timer() as t, pytest.raises(TimeoutError, match=r"from handler"): + with InterruptTimeout( + timeout_secs=timeout, + interruptions=[dict(sig=signal.SIGINT), dict(sig=signal.SIGTERM)], + before_interrupt=before, + ): + for i in range(100): + time.sleep(0.1) assert t.duration - timeout < 2 # default wait_retry_secs is 1s assert 15 < i < 25 assert before.call_count == 2 @@ -116,14 +113,13 @@ def _handler(*_): signal_handler(signal.SIGTERM, _handler), ): timeout = 1 - with Timer() as t: - with pytest.raises(TimeoutError, match=r"from handler"): - with InterruptTimeout( - timeout_secs=timeout, - interruptions=[dict(sig=signal.SIGINT), dict(sig=signal.SIGTERM)], - wait_retry_secs=0.3, - ): - for i in range(100): - time.sleep(0.1) + with Timer() as t, pytest.raises(TimeoutError, match=r"from handler"): + with InterruptTimeout( + timeout_secs=timeout, + interruptions=[dict(sig=signal.SIGINT), dict(sig=signal.SIGTERM)], + wait_retry_secs=0.3, + ): + for i in range(100): + time.sleep(0.1) assert t.duration - timeout < 1 assert 10 < i < 15 diff --git a/tests/unit/amlb/utils/serialization/test_serializers.py b/tests/unit/amlb/utils/serialization/test_serializers.py index 4e1cd4fde..9ec139fc6 100644 --- a/tests/unit/amlb/utils/serialization/test_serializers.py +++ b/tests/unit/amlb/utils/serialization/test_serializers.py @@ -3,7 +3,7 @@ import pytest from amlb.utils.core import Namespace as ns -from amlb.utils.serialization import is_sparse, serialize_data, deserialize_data +from amlb.utils.serialization import deserialize_data, is_sparse, serialize_data @pytest.mark.use_disk @@ -100,8 +100,8 @@ def test_serialize_pandas_dataframes(tmpdir): @pytest.mark.use_disk def test_serialize_sparse_matrix(tmpdir): - import scipy.sparse as sp import numpy as np + import scipy.sparse as sp arr = np.array([[0, 0, 0, 3.3], [4.4, 0, 0, 0], [0, np.nan, 0, 0]]) nans = np.count_nonzero(np.isnan(arr)) @@ -119,8 +119,8 @@ def test_serialize_sparse_matrix(tmpdir): @pytest.mark.use_disk def test_serialize_sparse_matrix_reload_as_dense(tmpdir): - import scipy.sparse as sp import numpy as np + import scipy.sparse as sp arr = np.array([[0, 0, 0, 3.3], [4.4, 0, 0, 0], [0, np.nan, 0, 0]]) mat = sp.csc_matrix(arr) @@ -139,8 +139,8 @@ def test_serialize_sparse_matrix_reload_as_dense(tmpdir): @pytest.mark.use_disk def test_serialize_sparse_matrix_reload_as_array(tmpdir): - import scipy.sparse as sp import numpy as np + import scipy.sparse as sp arr = np.array([[0, 0, 0, 3.3], [4.4, 0, 0, 0], [0, np.nan, 0, 0]]) mat = sp.csc_matrix(arr) diff --git a/upload_results.py b/upload_results.py index 310f020b8..b45587186 100644 --- a/upload_results.py +++ b/upload_results.py @@ -4,11 +4,11 @@ import argparse import contextlib -from contextlib import contextmanager -from datetime import datetime import logging import os import pathlib +from contextlib import contextmanager +from datetime import datetime from typing import Optional import openml @@ -16,7 +16,7 @@ from amlb.defaults import default_dirs from amlb.resources import config_load -from amlb.uploads import process_task_folder, missing_folds, _load_task_data +from amlb.uploads import _load_task_data, missing_folds, process_task_folder log = logging.getLogger(__name__) log.setLevel(logging.DEBUG)