Consolidate Python dependency definitions into a Pipfile-backed packaging flow

.github/workflows/comment-run.yml

@@ -22,9 +22,9 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v5
     - name: Install Test Dependencies
-      run: pip install -r test-requirements.txt
-    - name: Install IPWB from Source
-      run: pip install .
+      run: |
+        pip install --upgrade pip
+        pip install '.[test]'
     - name: Execute Code in Comment
       uses: ibnesayeed/actions-comment-run@master
       with:

.github/workflows/test.yml

@@ -42,8 +42,7 @@ jobs:
         shell: bash
         run: |
           pip install --upgrade pip
-          pip install -r requirements.txt
-          pip install -r test-requirements.txt
+          pip install '.[test]'
       - name: Run Tests
         shell: bash
         run: py.test -s --cov=./

Pipfile

@@ -0,0 +1,23 @@
+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[packages]
+warcio = ">=1.5.3"
+ipfshttpclient = ">=0.8.0a"
+Flask = ">=3.0"
+pycryptodome = ">=3.4.11"
+requests = ">=2.19.1"
+beautifulsoup4 = ">=4.6.3"
+surt = ">=0.3.0"
+multiaddr = ">=0.0.9"
+packaging = "==23.0"
+
+[dev-packages]
+flake8 = ">=3.7.9"
+pycodestyle = "*"
+pytest = ">=5.3.5,<9"
+pytest-cov = "*"
+pytest-flake8 = "*"
+tomli = ">=2.0.1"

README.md

@@ -39,6 +39,14 @@ $ cd ipwb
 $ pip install ./
 ```
 
+Developers who want a managed local environment can use Pipenv with the
+repository's `Pipfile`:
+
+```
+$ pip install pipenv
+$ pipenv install --dev
+```
+
 ## Setup
 
 The InterPlanetary File System (IPFS) daemon (named "kubo", previously "go-ipfs") must be installed and running before starting ipwb. [Download kubo](https://dist.ipfs.tech/#kubo) and [take your node online](https://docs.ipfs.tech/how-to/command-line-quick-start/#take-your-node-online) to start the IPFS daemon. Once installed, this can be done using the command:

pyproject.toml

@@ -0,0 +1,6 @@
+[build-system]
+requires = [
+    "setuptools>=78.1.1",
+    "tomli>=2.0.1; python_version < '3.11'",
+]
+build-backend = "setuptools.build_meta"

setup.py

@@ -1,80 +1,105 @@
 #!/usr/bin/env python
 
+from functools import lru_cache
+from pathlib import Path
+
 from setuptools import setup
-from ipwb import __version__
 
-with open('README.md') as f:
-    long_description = f.read()
+try:
+    import tomllib
+except ModuleNotFoundError:
+    import tomli as tomllib
+
+ROOT = Path(__file__).resolve().parent
+PACKAGE_INIT = ROOT / 'ipwb' / '__init__.py'
+PIPFILE = ROOT / 'Pipfile'
+README = ROOT / 'README.md'
 desc = """InterPlanetary Wayback (ipwb): Web Archive integration with IPFS"""
 
-setup(
-    name='ipwb',
-    version=__version__,
-    url='https://github.com/oduwsdl/ipwb',
-    download_url="https://github.com/oduwsdl/ipwb",
-    author='Mat Kelly',
-    author_email='me@matkelly.com',
-    description=desc,
-    packages=['ipwb'],
-    python_requires='>=3.9',
-    license='MIT',
-    long_description=long_description,
-    long_description_content_type="text/markdown",
-    provides=[
-        'ipwb'
-    ],
-    install_requires=[
-        'warcio>=1.5.3',
-        'ipfshttpclient>=0.8.0a',
-        'Flask>=3.0',
-        'pycryptodome>=3.4.11',
-        'requests>=2.19.1',
-        'beautifulsoup4>=4.6.3',
-        'surt>=0.3.0'
-    ],
-    tests_require=[
-        'flake8>=3.4',
-        'pytest>=3.6',
-        'pytest-cov',
-        'pytest-flake8'
-    ],
-    entry_points="""
-        [console_scripts]
-        ipwb = ipwb.__main__:main
-    """,
-    package_data={
-        'ipwb': [
-            'assets/*.*',
-            'assets/favicons/*.*',
-            'templates/*.*'
-          ]
-    },
-    zip_safe=False,
-    keywords='http web archives ipfs distributed odu wayback memento',
-    classifiers=[
-        'Development Status :: 4 - Beta',
-
-        'Environment :: Web Environment',
-
-        'Programming Language :: Python :: 3.9',
-        'Programming Language :: Python :: 3.10',
-        'Programming Language :: Python :: 3.11',
-        'Programming Language :: Python :: 3.12',
-        'Programming Language :: Python :: 3.13',
-
-        'License :: OSI Approved :: MIT License',
-
-        'Intended Audience :: Developers',
-        'Intended Audience :: Information Technology',
-        'Intended Audience :: Science/Research',
-
-        'Topic :: Internet :: WWW/HTTP',
-        'Topic :: System :: Archiving',
-        'Topic :: System :: Archiving :: Backup',
-        'Topic :: System :: Archiving :: Mirroring',
-        'Topic :: Utilities',
+
+def _load_version():
+    for line in PACKAGE_INIT.read_text().splitlines():
+        if line.startswith('__version__ = '):
+            return line.split('=', 1)[1].strip().strip("'\"")
+    raise RuntimeError('Unable to determine package version.')
+
+
+@lru_cache(maxsize=1)
+def _load_pipfile():
+    with PIPFILE.open('rb') as fh:
+        return tomllib.load(fh)
+
+
+def _load_requirements(group):
+    return [
+        package if specifier == '*' else f'{package}{specifier}'
+        for package, specifier in _load_pipfile()[group].items()
     ]
-)
+
+
+def get_setup_kwargs():
+    return {
+        'name': 'ipwb',
+        'version': _load_version(),
+        'url': 'https://github.com/oduwsdl/ipwb',
+        'download_url': "https://github.com/oduwsdl/ipwb",
+        'author': 'Mat Kelly',
+        'author_email': 'me@matkelly.com',
+        'description': desc,
+        'packages': ['ipwb'],
+        'python_requires': '>=3.9',
+        'license': 'MIT',
+        'long_description': README.read_text(),
+        'long_description_content_type': "text/markdown",
+        'provides': [
+            'ipwb'
+        ],
+        'install_requires': _load_requirements('packages'),
+        'tests_require': _load_requirements('dev-packages'),
+        'extras_require': {
+            'test': _load_requirements('dev-packages')
+        },
+        'entry_points': """
+            [console_scripts]
+            ipwb = ipwb.__main__:main
+        """,
+        'package_data': {
+            'ipwb': [
+                'assets/*.*',
+                'assets/favicons/*.*',
+                'templates/*.*'
+              ]
+        },
+        'zip_safe': False,
+        'keywords': 'http web archives ipfs distributed odu wayback memento',
+        'classifiers': [
+            'Development Status :: 4 - Beta',
+
+            'Environment :: Web Environment',
+
+            'Programming Language :: Python :: 3.9',
+            'Programming Language :: Python :: 3.10',
+            'Programming Language :: Python :: 3.11',
+            'Programming Language :: Python :: 3.12',
+            'Programming Language :: Python :: 3.13',
+
+            'License :: OSI Approved :: MIT License',
+
+            'Intended Audience :: Developers',
+            'Intended Audience :: Information Technology',
+            'Intended Audience :: Science/Research',
+
+            'Topic :: Internet :: WWW/HTTP',
+            'Topic :: System :: Archiving',
+            'Topic :: System :: Archiving :: Backup',
+            'Topic :: System :: Archiving :: Mirroring',
+            'Topic :: Utilities',
+        ]
+    }
+
+
+if __name__ == '__main__':
+    setup(**get_setup_kwargs())
 
 # Publish to pypi:
 #   rm -rf dist; python setup.py sdist bdist_wheel; twine upload dist/*

tests/test_setup.py

@@ -0,0 +1,39 @@
+import importlib.util
+from pathlib import Path
+
+try:
+    import tomllib
+except ModuleNotFoundError:
+    import tomli as tomllib
+
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def _load_setup_module():
+    spec = importlib.util.spec_from_file_location('setup_module', ROOT / 'setup.py')
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def test_setup_uses_pipfile_runtime_dependencies():
+    setup_module = _load_setup_module()
+    with (ROOT / 'Pipfile').open('rb') as fh:
+        pipfile = tomllib.load(fh)
+
+    assert setup_module.get_setup_kwargs()['install_requires'] == [
+        package if specifier == '*' else f'{package}{specifier}'
+        for package, specifier in pipfile['packages'].items()
+    ]
+
+
+def test_setup_uses_pipfile_dev_dependencies():
+    setup_module = _load_setup_module()
+    with (ROOT / 'Pipfile').open('rb') as fh:
+        pipfile = tomllib.load(fh)
+
+    assert setup_module.get_setup_kwargs()['tests_require'] == [
+        package if specifier == '*' else f'{package}{specifier}'
+        for package, specifier in pipfile['dev-packages'].items()
+    ]