diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ef74e9e..38c146d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -32,7 +32,7 @@ jobs: ${{ matrix.extra-pytest-warnings }} enable-coverage: ${{ matrix.enable-coverage }} jupyter-platform-dirs: "1" - package-managers: "uv pixi conda" + package-managers: "uv poetry pixi conda" codecov-flags: "unit" strategy: fail-fast: false diff --git a/doc/howtoguides/requirements.rst b/doc/howtoguides/requirements.rst index 4c360f8..74867f6 100644 --- a/doc/howtoguides/requirements.rst +++ b/doc/howtoguides/requirements.rst @@ -19,8 +19,8 @@ What is stored :term:`package manager` can recreate the environment from this list. * ``manager``: the :term:`package manager` that generated the requirements, with the content of the files it needs to recreate the environment: ``requirements.txt`` for pip-venv, - ``pyproject.toml`` and ``uv.lock`` for uv, ``environment.yml`` for conda, ``pixi.toml`` - and ``pixi.lock`` for pixi. + ``pyproject.toml`` and ``uv.lock`` for uv, ``pyproject.toml`` and ``poetry.lock`` for + poetry, ``environment.yml`` for conda, ``pixi.toml`` and ``pixi.lock`` for pixi. Use ``--exclude-requirements`` to store nothing. @@ -88,17 +88,18 @@ content. Without ``--env-root`` the :term:`package manager` decides where the environment goes: conda creates it in its own environment directory (``conda config --show envs_dirs``), so -``conda activate `` finds it. venv, uv and pixi create an environment wherever they are -told to, so for those ewoks uses ``~/.ewoks/envs``. +``conda activate `` finds it, and poetry in the directory it creates project +environments in (``poetry config virtualenvs.path``). venv, uv and pixi create an environment +wherever they are told to, so for those ewoks uses ``~/.ewoks/envs``. An environment that already exists is installed in, which adds the requirements to what is already there. Use ``--clean`` to remove it first. Only a directory that contains a python environment is removed. The environment of a :term:`workflow` is a directory. For pip-venv and conda it is a python -environment, for uv it is a project with the environment in ``.venv`` and for pixi it is a -workspace with the environment in ``.pixi/envs/default``. ``ewoks execute --env`` takes that -directory, not the python interpreter inside it. +environment, for uv and poetry it is a project with the environment in ``.venv`` and for pixi +it is a workspace with the environment in ``.pixi/envs/default``. ``ewoks execute --env`` takes +that directory, not the python interpreter inside it. Limitations ----------- @@ -106,14 +107,15 @@ Limitations * ``--in-place`` is only supported by pip-venv and uv. The other package managers only install in an environment they created themselves. * ``--python-version`` is a request: uv can provide any python version, conda and pixi provide - the patch versions built by their channel and pip-venv can only use the version of the python - interpreter that creates the environment. A warning is emitted when the version cannot be - provided. -* uv and pixi resolve the ``distributions`` into a lock file, which requires access to the - package index. When that fails, a warning is emitted and the requirements are stored without - files: the environment is then recreated from the ``distributions`` list. + the patch versions built by their channel, and pip-venv and poetry can only use the version of + the python interpreter that creates the environment. A warning is emitted when the version + cannot be provided. +* uv, poetry and pixi resolve the ``distributions`` into a lock file, which requires access to + the package index. When that fails, a warning is emitted and the requirements are stored + without files: the environment is then recreated from the ``distributions`` list. * A lock file is only read by a :term:`package manager` that understands its format version, so reproducing an environment can require a version of the tool that is at least as recent as the one that generated the requirements. +* poetry 1.8 or later is required. Older versions are reported as not available. * A package installed from a local directory cannot be recreated elsewhere. This is reported as a warning when the requirements are generated. diff --git a/doc/reference/glossary.rst b/doc/reference/glossary.rst index 4c3b11f..68b2b2e 100644 --- a/doc/reference/glossary.rst +++ b/doc/reference/glossary.rst @@ -29,7 +29,7 @@ Glossary An execution engine is the underlying software used to execute the :term:`workflow`. :term:`Ewoks` supports multiple execution engines: pypushflow, orange, dask and the ewoks internal excution engine. Package manager - A package manager creates python environments and installs packages in them. :term:`Ewoks` uses package managers to store the environment in which a :term:`workflow` was created and to recreate it: pip with venv, uv, conda and pixi. + A package manager creates python environments and installs packages in them. :term:`Ewoks` uses package managers to store the environment in which a :term:`workflow` was created and to recreate it: pip with venv, uv, poetry, conda and pixi. blissdata `Blissdata `_ is an API for accessing data from BLISS in memory. diff --git a/doc/tutorials/install.rst b/doc/tutorials/install.rst index de89948..96dd6f7 100644 --- a/doc/tutorials/install.rst +++ b/doc/tutorials/install.rst @@ -22,6 +22,7 @@ of the supported ones install/pip_venv install/uv + install/poetry install/conda install/pixi diff --git a/doc/tutorials/install/poetry.rst b/doc/tutorials/install/poetry.rst new file mode 100644 index 0000000..f4360bd --- /dev/null +++ b/doc/tutorials/install/poetry.rst @@ -0,0 +1,153 @@ +.. _install_poetry: + +poetry +====== + +End-to-end walk-through with `poetry `_ 1.8 or later. Poetry works +with projects instead of environments: its commands need a directory with a +``pyproject.toml`` file, selected with ``-C``. + +Install poetry +-------------- + +.. tabs:: + + .. group-tab:: Linux + + .. code-block:: bash + + export POETRY_HOME=$HOME/.local/poetry + curl -sSL https://install.python-poetry.org | python3 - + export PATH=$POETRY_HOME/bin:$PATH + + .. group-tab:: macOS + + .. code-block:: bash + + export POETRY_HOME=$HOME/.local/poetry + curl -sSL https://install.python-poetry.org | python3 - + export PATH=$POETRY_HOME/bin:$PATH + + .. group-tab:: Windows + + .. code-block:: powershell + + $env:POETRY_HOME = "$env:USERPROFILE\.local\poetry" + (Invoke-WebRequest -Uri https://install.python-poetry.org -UseBasicParsing).Content | py - + $env:PATH = "$env:POETRY_HOME\bin;$env:PATH" + +Producer side +------------- + +Create a project, install :term:`ewoks` in it and store a :term:`workflow` with the packages +of that project as its ``requirements`` + +.. tabs:: + + .. group-tab:: Linux + + .. code-block:: bash + + poetry new ewoks_producer + poetry -C ewoks_producer add ewoks + poetry -C ewoks_producer run ewoks convert demo "$PWD/demo.json" --test + + .. group-tab:: macOS + + .. code-block:: bash + + poetry new ewoks_producer + poetry -C ewoks_producer add ewoks + poetry -C ewoks_producer run ewoks convert demo "$PWD/demo.json" --test + + .. group-tab:: Windows + + .. code-block:: powershell + + poetry new ewoks_producer + poetry -C ewoks_producer add ewoks + poetry -C ewoks_producer run ewoks convert demo "$PWD\demo.json" --test + +Install any package that provides :term:`tasks ` instead of +``ewoks`` for a real :term:`workflow`. Poetry 2 resolves the arguments of ``poetry run`` +relative to the project directory, so the workflow paths are absolute. + +Re-producer side +---------------- + +Only ``ewoks`` itself is needed to recreate the environment of ``demo.json`` + +.. tabs:: + + .. group-tab:: Linux + + .. code-block:: bash + + poetry new ewoks_reproducer + poetry -C ewoks_reproducer add ewoks + poetry -C ewoks_reproducer run ewoks install "$PWD/demo.json" --yes --env-root "$PWD/ewoks_envs" + + .. group-tab:: macOS + + .. code-block:: bash + + poetry new ewoks_reproducer + poetry -C ewoks_reproducer add ewoks + poetry -C ewoks_reproducer run ewoks install "$PWD/demo.json" --yes --env-root "$PWD/ewoks_envs" + + .. group-tab:: Windows + + .. code-block:: powershell + + poetry new ewoks_reproducer + poetry -C ewoks_reproducer add ewoks + poetry -C ewoks_reproducer run ewoks install "$PWD\demo.json" --yes --env-root "$PWD\ewoks_envs" + +The environment of the :term:`workflow` is a poetry project in ``ewoks_envs/demo`` with the +virtual environment in its ``.venv`` directory. + +Execute the workflow +-------------------- + +.. tabs:: + + .. group-tab:: Linux + + .. code-block:: bash + + poetry -C ewoks_reproducer run ewoks execute --env "$PWD/ewoks_envs/demo" "$PWD/demo.json" --outputs=all + + .. group-tab:: macOS + + .. code-block:: bash + + poetry -C ewoks_reproducer run ewoks execute --env "$PWD/ewoks_envs/demo" "$PWD/demo.json" --outputs=all + + .. group-tab:: Windows + + .. code-block:: powershell + + poetry -C ewoks_reproducer run ewoks execute --env "$PWD\ewoks_envs\demo" "$PWD\demo.json" --outputs=all + +Clean up +-------- + +.. tabs:: + + .. group-tab:: Linux + + .. code-block:: bash + + rm -rf ewoks_producer ewoks_reproducer ewoks_envs demo.json + + .. group-tab:: macOS + + .. code-block:: bash + + rm -rf ewoks_producer ewoks_reproducer ewoks_envs demo.json + + .. group-tab:: Windows + + .. code-block:: powershell + + Remove-Item -Recurse -Force ewoks_producer, ewoks_reproducer, ewoks_envs, demo.json diff --git a/src/ewoks/_requirements/poetry.py b/src/ewoks/_requirements/poetry.py new file mode 100644 index 0000000..e5dfa35 --- /dev/null +++ b/src/ewoks/_requirements/poetry.py @@ -0,0 +1,239 @@ +import json +import logging +import os +import platform +import re +import shutil +import sys +from pathlib import Path +from typing import Dict +from typing import Literal +from typing import Mapping +from typing import Optional +from typing import Sequence +from typing import Tuple +from typing import Union + +from pydantic import Field + +from .utils import requirements_txt +from .utils import toml_dependencies +from .utils.base_manager import BaseManager +from .utils.base_manager import BaseManagerInfo +from .utils.base_manager import BaseRequirements +from .utils.environment import Environment +from .utils.environment import interpreter_version +from .utils.files import read_files +from .utils.files import temporary_files +from .utils.files import write_files +from .utils.metadata import models + +logger = logging.getLogger(__name__) + +_MINIMUM_VERSION = (1, 8) + +_PYPROJECT_FILENAME = "pyproject.toml" +_LOCK_FILENAME = "poetry.lock" + + +class PoetryManagerInfo(BaseManagerInfo): + name: Literal["poetry"] = Field( + default="poetry", + description="Environments described and created by a `poetry` project.", + examples=["poetry"], + ) + + +class PoetryRequirements(BaseRequirements): + manager: PoetryManagerInfo + + +class PoetryManager(BaseManager): + """Uses a poetry project: `poetry lock` describes an environment with a + `pyproject.toml` and a `poetry.lock` file, `poetry install` reproduces the + environment from those files. + + The project is generated from the installed python distributions instead of + being taken from a poetry project of the current environment. This keeps it + self-contained: no local project to build. + + Environments are created with `poetry env use`, which needs an interpreter of + the requested python version because poetry cannot install one. + + The command invokes poetry. For example `PoetryManager("/path/to/poetry")`. + """ + + NAME = "poetry" + PRIORITY = 2 + REQUIREMENTS_MODEL = PoetryRequirements + COMMAND_EXAMPLE = "poetry" + ENVIRONMENT_SUBDIR = ".venv" + + def __init__(self, *command: str) -> None: + if not command: + command = ("poetry",) + super().__init__(*command) + + def version(self) -> Optional[str]: + """Returns None when this manager is not available.""" + try: + output = self._check_output( + "--version", extra_env=_python_env(sys.executable) + ) + except RuntimeError: + return None + # For example "Poetry (version 2.4.1)" + match = re.search(r"(\d+\.\d+(?:\.\d+)*)", output) + if not match: + return None + version = match.group(1) + if _version_tuple(version) < _MINIMUM_VERSION: + # Older versions have no project without a package to build + logger.warning( + "Poetry %s is not supported: version %s or later is required", + version, + ".".join(str(part) for part in _MINIMUM_VERSION), + ) + return None + return version + + def is_active(self) -> bool: + """Manager is explicitly active.""" + return "POETRY_ACTIVE" in os.environ or _is_project(Path(sys.prefix).parent) + + def create_environment( + self, location: Path, python_version: Optional[str] = None + ) -> Environment: + """ + :raises RuntimeError: creation failed + """ + # The project that poetry attaches the environment to. Its python version + # is not constrained yet: the requirements provide it when they are installed. + write_files(location, {_PYPROJECT_FILENAME: _pyproject([], "")}) + self._check_call( + "-C", + location, + "env", + "use", + _base_python(python_version), + extra_env=_python_env(sys.executable), + ) + return self.environment(location) + + def environments_root(self) -> Path: + """Directory in which poetry creates the environments of projects that do + not contain them (`virtualenvs.path`). + """ + try: + root = Path( + self._check_output( + "config", "virtualenvs.path", extra_env=_python_env(sys.executable) + ).strip() + ) + if not root.is_absolute(): + raise RuntimeError(f"{str(root)!r} is not a directory") + return root + except RuntimeError as ex: + logger.warning( + "Cannot determine where poetry creates environments (%s)", ex + ) + return super().environments_root() + + def _files_from_distributions( + self, distributions: Sequence[models.Distribution], python_version: str + ) -> Dict[str, str]: + """ + :raises RuntimeError: the requirements cannot be resolved + """ + pyproject = _pyproject( + requirements_txt.manifest_requirements(distributions), python_version + ) + files = {_PYPROJECT_FILENAME: pyproject} + + with temporary_files(files) as directory: + self._check_call( + "-C", directory, "lock", extra_env=_python_env(sys.executable) + ) + lock = read_files(directory, _LOCK_FILENAME) + + return {**files, **lock} + + def _install_files( + self, files: Mapping[str, str], environment: Environment + ) -> None: + write_files(environment.location, files) + self._check_call( + "-C", + environment.location, + "install", + "--no-root", + extra_env=_python_env(environment.python), + ) + + def _add_ewoks( + self, requirements: BaseRequirements, environment: Environment + ) -> None: + # The project pins all versions so they cannot change + self._check_call( + "-C", + environment.location, + "add", + "ewoks", + extra_env=_python_env(environment.python), + ) + + +def _pyproject(requirements: Sequence[str], python_version: str) -> str: + """Content of a `pyproject.toml` file for a project that is not a package. + + The dependencies are declared in the `tool.poetry` section instead of a PEP 621 + `project` section, which poetry only understands since version 2. + """ + python = f"=={python_version}" if python_version else "*" + dependencies = "".join( + f"{line}\n" for line in toml_dependencies.dependency_lines(requirements) + ) + return f"""[tool.poetry] +package-mode = false + +[tool.poetry.dependencies] +python = {json.dumps(python)} +{dependencies}""" + + +def _base_python(python_version: Optional[str]) -> Path: + """Interpreter poetry creates an environment with.""" + if not python_version or python_version == platform.python_version(): + return Path(sys.executable) + major_minor = ".".join(python_version.split(".")[:2]) + python = Path(shutil.which(f"python{major_minor}") or sys.executable) + version = interpreter_version(python) + if version != python_version: + logger.warning( + "poetry cannot provide python %s: creating the environment with " + "python %s instead", + python_version, + version, + ) + return python + + +def _version_tuple(version: str) -> Tuple[int, ...]: + return tuple(int(part) for part in version.split(".") if part.isdigit()) + + +def _python_env(python: Union[str, Path]) -> Dict[str, str]: + """Poetry searches `PATH` for a python interpreter, which is not necessarily + a working one. Make sure it finds this interpreter first. The environment of + a project is the `.venv` directory inside it. + """ + path = os.environ.get("PATH", "") + return { + "PATH": os.pathsep.join([str(Path(python).parent), path]), + "POETRY_VIRTUALENVS_IN_PROJECT": "true", + } + + +def _is_project(directory: Path) -> bool: + """Directory is a poetry project.""" + return (directory / _LOCK_FILENAME).is_file() diff --git a/src/ewoks/_requirements/utils/_supported.py b/src/ewoks/_requirements/utils/_supported.py index 9eb155e..11fc6e6 100644 --- a/src/ewoks/_requirements/utils/_supported.py +++ b/src/ewoks/_requirements/utils/_supported.py @@ -5,6 +5,7 @@ from ..conda import CondaManager from ..pip_venv import PipVenvManager from ..pixi import PixiManager +from ..poetry import PoetryManager from ..uv import UvManager from .base_manager import BaseManager @@ -14,6 +15,7 @@ def get_supported_managers() -> Dict[str, Type[BaseManager]]: managers = [ PipVenvManager, UvManager, + PoetryManager, CondaManager, PixiManager, ] diff --git a/src/ewoks/_requirements/utils/base_manager.py b/src/ewoks/_requirements/utils/base_manager.py index 82e33eb..31d15cb 100644 --- a/src/ewoks/_requirements/utils/base_manager.py +++ b/src/ewoks/_requirements/utils/base_manager.py @@ -26,7 +26,7 @@ class BaseManagerInfo(models.BaseModel): name: str = Field( description="Package manager that generated the requirements.", - examples=["pip-venv", "uv", "conda", "pixi"], + examples=["pip-venv", "uv", "poetry", "conda", "pixi"], ) version: str = Field( description="Version of the package manager.", examples=["25.0.1"] @@ -36,9 +36,9 @@ class BaseManagerInfo(models.BaseModel): description=( "Content of the files the package manager needs to reproduce the " "environment: 'requirements.txt' for pip-venv, 'pyproject.toml' and " - "'uv.lock' for uv, 'environment.yml' for conda, 'pixi.toml' and " - "'pixi.lock' for pixi. Empty when the package manager could not " - "generate them." + "'uv.lock' for uv, 'pyproject.toml' and 'poetry.lock' for poetry, " + "'environment.yml' for conda, 'pixi.toml' and 'pixi.lock' for pixi. " + "Empty when the package manager could not generate them." ), examples=[{"requirements.txt": "ewoks==7.0.0\nnetworkx==3.4.2\n"}], ) @@ -150,7 +150,7 @@ def installed_distribution(cls, distribution: models.Distribution) -> bool: @staticmethod def _installer(distribution: models.Distribution) -> str: """Tool that installed the distribution. It can contain more than the name - of the tool, for example its version. + of the tool: poetry writes "Poetry 1.8.5". """ return (distribution.installer or "").lower() diff --git a/src/ewoks/_requirements/utils/environment.py b/src/ewoks/_requirements/utils/environment.py index 6c0d853..4f82d31 100644 --- a/src/ewoks/_requirements/utils/environment.py +++ b/src/ewoks/_requirements/utils/environment.py @@ -13,7 +13,7 @@ ENVIRONMENT_SUBDIRS: Tuple[str, ...] = ( "", # environment prefix, for example created by `python -m venv` - ".venv", # uv project + ".venv", # uv and poetry project os.path.join(".pixi", "envs", "default"), # pixi project ) diff --git a/src/ewoks/_requirements/utils/toml_dependencies.py b/src/ewoks/_requirements/utils/toml_dependencies.py index 38d2517..2591992 100644 --- a/src/ewoks/_requirements/utils/toml_dependencies.py +++ b/src/ewoks/_requirements/utils/toml_dependencies.py @@ -1,5 +1,6 @@ """Dependencies of a TOML manifest: `name = spec` entries in which the -specification is a version constraint, an URL or a git reference. +specification is a version constraint, an URL or a git reference. Poetry and pixi +use the same syntax. """ import json diff --git a/src/ewoks/tests/requirements/conftest.py b/src/ewoks/tests/requirements/conftest.py index 686b156..cb50a1a 100644 --- a/src/ewoks/tests/requirements/conftest.py +++ b/src/ewoks/tests/requirements/conftest.py @@ -51,6 +51,7 @@ def _home_variables(home: Path) -> Dict[str, Path]: "CONDA_PKGS_DIRS": home / "conda" / "pkgs", # Directories in which package managers create named environments "CONDA_ENVS_DIRS": home / "conda" / "envs", + "POETRY_VIRTUALENVS_PATH": home / "poetry" / "virtualenvs", # Channels are no longer provided by the home directory of the user "CONDARC": condarc, } diff --git a/src/ewoks/tests/requirements/managers.py b/src/ewoks/tests/requirements/managers.py index af07e58..74dd463 100644 --- a/src/ewoks/tests/requirements/managers.py +++ b/src/ewoks/tests/requirements/managers.py @@ -14,6 +14,7 @@ from ..._requirements.conda import CondaManager from ..._requirements.pip_venv import PipVenvManager from ..._requirements.pixi import PixiManager +from ..._requirements.poetry import PoetryManager from ..._requirements.utils import conda_channel from ..._requirements.utils.base_manager import BaseManager from ..._requirements.utils.metadata import models @@ -84,6 +85,23 @@ def native_files( return dict() +class PoetryCase(ManagerCase): + NAME = "poetry" + MANAGER_CLS = PoetryManager + INSTALLER = "Poetry 1.8.5" # poetry adds its version + + def native_files( + self, distributions: Sequence[models.Distribution], python_version: str + ) -> Dict[str, str]: + # Resolving a lock file requires the package index + try: + return PoetryManager()._files_from_distributions( + distributions, python_version + ) + except RuntimeError: + return dict() + + class CondaCase(ManagerCase): NAME = "conda" MANAGER_CLS = CondaManager @@ -130,6 +148,7 @@ def native_files( MANAGER_CASES: List[ManagerCase] = [ PipVenvCase(), UvCase(), + PoetryCase(), CondaCase(), PixiCase(), ] diff --git a/src/ewoks/tests/requirements/test_poetry.py b/src/ewoks/tests/requirements/test_poetry.py new file mode 100644 index 0000000..1654c9b --- /dev/null +++ b/src/ewoks/tests/requirements/test_poetry.py @@ -0,0 +1,74 @@ +"""Tests specific to the poetry package manager.""" + +import sys + +import pytest + +from ..._requirements.poetry import PoetryManager +from ..._requirements.poetry import _pyproject +from ..._requirements.utils.base_manager import EWOKS_ENVIRONMENTS_ROOT +from .utils import DISTRIBUTIONS +from .utils import PYTHON_VERSION +from .utils import REQUIREMENTS + + +def test_pyproject(): + """The project is not a package: there is nothing to build. Dependencies are + declared in the legacy format, which every supported poetry version reads.""" + pyproject = _pyproject(REQUIREMENTS, PYTHON_VERSION) + + assert "package-mode = false" in pyproject + assert f'python = "=={PYTHON_VERSION}"' in pyproject + for dist in DISTRIBUTIONS: + assert f'"{dist.name}" = "=={dist.version}"' in pyproject + assert "[project]" not in pyproject + + +def test_project_environment(monkeypatch, tmp_path): + """The environment of a poetry project is the `.venv` directory.""" + location = tmp_path / "project" + prefix = location / ".venv" + + assert PoetryManager().environment(location).prefix == prefix + + monkeypatch.delenv("POETRY_ACTIVE", raising=False) + monkeypatch.setattr(sys, "prefix", str(prefix)) + assert not PoetryManager().is_active() + + location.mkdir() + with open(location / "poetry.lock", "w", encoding="utf-8") as fh: + fh.write("# lock\n") + assert PoetryManager().is_active() + + +def test_is_active_poetry_run(monkeypatch, tmp_path): + monkeypatch.setattr(sys, "prefix", str(tmp_path)) + monkeypatch.setenv("POETRY_ACTIVE", "1") + assert PoetryManager().is_active() + + +def test_version_not_available(): + assert PoetryManager("poetry-does-not-exist").version() is None + + +@pytest.mark.skipif(PoetryManager().version() is None, reason="poetry is not installed") +def test_version(): + version = PoetryManager().version() + assert version[0].isdigit() + + +@pytest.mark.skipif(PoetryManager().version() is None, reason="poetry is not installed") +def test_environments_root(monkeypatch, tmp_path): + """Named environments are created where poetry creates them itself.""" + monkeypatch.setenv("POETRY_VIRTUALENVS_PATH", str(tmp_path)) + + assert PoetryManager().environments_root() == tmp_path + + +def test_environments_root_not_available(caplog): + """Environments of ewoks when poetry cannot provide its environment directory.""" + manager = PoetryManager("poetry-does-not-exist") + + assert manager.environments_root() == EWOKS_ENVIRONMENTS_ROOT + + assert "Cannot determine where poetry creates environments" in caplog.text