From 9b2caa633fe093767ba8b4e79e4355696be6c674 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 15 Jun 2026 17:34:06 +0000 Subject: [PATCH 1/4] Initial plan From e39169119c8a19440c58626ebb836f4f537bc911 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 15 Jun 2026 17:38:21 +0000 Subject: [PATCH 2/4] Use Pipfile as dependency source --- .github/workflows/comment-run.yml | 16 +++- .github/workflows/test.yml | 16 +++- Pipfile | 22 +++++ README.md | 8 ++ requirements.txt | 10 -- setup.py | 153 ++++++++++++++++-------------- test-requirements.txt | 5 - tests/test_setup.py | 40 ++++++++ 8 files changed, 183 insertions(+), 87 deletions(-) create mode 100644 Pipfile delete mode 100644 requirements.txt delete mode 100644 test-requirements.txt create mode 100644 tests/test_setup.py diff --git a/.github/workflows/comment-run.yml b/.github/workflows/comment-run.yml index 2afc91b9..ed6b4034 100644 --- a/.github/workflows/comment-run.yml +++ b/.github/workflows/comment-run.yml @@ -22,7 +22,21 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 - name: Install Test Dependencies - run: pip install -r test-requirements.txt + run: | + pip install --upgrade pip + python - <<'PY' + import subprocess + import sys + from setup import get_setup_kwargs + + subprocess.check_call([ + sys.executable, + '-m', + 'pip', + 'install', + *get_setup_kwargs()['tests_require'], + ]) + PY - name: Install IPWB from Source run: pip install . - name: Execute Code in Comment diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e656cce7..38600bfc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -42,8 +42,20 @@ jobs: shell: bash run: | pip install --upgrade pip - pip install -r requirements.txt - pip install -r test-requirements.txt + pip install . + python - <<'PY' + import subprocess + import sys + from setup import get_setup_kwargs + + subprocess.check_call([ + sys.executable, + '-m', + 'pip', + 'install', + *get_setup_kwargs()['tests_require'], + ]) + PY - name: Run Tests shell: bash run: py.test -s --cov=./ diff --git a/Pipfile b/Pipfile new file mode 100644 index 00000000..3d6255dd --- /dev/null +++ b/Pipfile @@ -0,0 +1,22 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[packages] +warcio = ">=1.5.3" +ipfshttpclient = ">=0.8.0a" +Flask = ">=3.0" +pycryptodome = ">=3.4.11" +requests = ">=2.19.1" +beautifulsoup4 = ">=4.6.3" +surt = ">=0.3.0" +multiaddr = ">=0.0.9" +packaging = "==23.0" + +[dev-packages] +flake8 = ">=3.7.9" +pycodestyle = "*" +pytest = ">=5.3.5,<9" +pytest-cov = "*" +pytest-flake8 = "*" diff --git a/README.md b/README.md index bb9fa0bc..f18e804f 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,14 @@ $ cd ipwb $ pip install ./ ``` +Developers who want a managed local environment can use Pipenv with the +repository's `Pipfile`: + +``` +$ pip install pipenv +$ pipenv install --dev +``` + ## Setup The InterPlanetary File System (IPFS) daemon (named "kubo", previously "go-ipfs") must be installed and running before starting ipwb. [Download kubo](https://dist.ipfs.tech/#kubo) and [take your node online](https://docs.ipfs.tech/how-to/command-line-quick-start/#take-your-node-online) to start the IPFS daemon. Once installed, this can be done using the command: diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index cb1b78df..00000000 --- a/requirements.txt +++ /dev/null @@ -1,10 +0,0 @@ -warcio>=1.5.3 -ipfshttpclient>=0.8.0a -Flask>=3.0 -pycryptodome>=3.4.11 -requests>=2.19.1 -beautifulsoup4>=4.6.3 -surt>=0.3.0 -multiaddr >= 0.0.9 -packaging==23.0 -setuptools==75.2.0 diff --git a/setup.py b/setup.py index db633d88..60b6d50b 100644 --- a/setup.py +++ b/setup.py @@ -1,80 +1,95 @@ #!/usr/bin/env python +from pathlib import Path + from setuptools import setup + from ipwb import __version__ -with open('README.md') as f: - long_description = f.read() +try: + import tomllib +except ModuleNotFoundError: + try: + import tomli as tomllib + except ModuleNotFoundError: + from pip._vendor import tomli as tomllib + +ROOT = Path(__file__).resolve().parent +PIPFILE = ROOT / 'Pipfile' +README = ROOT / 'README.md' desc = """InterPlanetary Wayback (ipwb): Web Archive integration with IPFS""" -setup( - name='ipwb', - version=__version__, - url='https://github.com/oduwsdl/ipwb', - download_url="https://github.com/oduwsdl/ipwb", - author='Mat Kelly', - author_email='me@matkelly.com', - description=desc, - packages=['ipwb'], - python_requires='>=3.9', - license='MIT', - long_description=long_description, - long_description_content_type="text/markdown", - provides=[ - 'ipwb' - ], - install_requires=[ - 'warcio>=1.5.3', - 'ipfshttpclient>=0.8.0a', - 'Flask>=3.0', - 'pycryptodome>=3.4.11', - 'requests>=2.19.1', - 'beautifulsoup4>=4.6.3', - 'surt>=0.3.0' - ], - tests_require=[ - 'flake8>=3.4', - 'pytest>=3.6', - 'pytest-cov', - 'pytest-flake8' - ], - entry_points=""" - [console_scripts] - ipwb = ipwb.__main__:main - """, - package_data={ - 'ipwb': [ - 'assets/*.*', - 'assets/favicons/*.*', - 'templates/*.*' - ] - }, - zip_safe=False, - keywords='http web archives ipfs distributed odu wayback memento', - classifiers=[ - 'Development Status :: 4 - Beta', - - 'Environment :: Web Environment', - - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', - 'Programming Language :: Python :: 3.12', - 'Programming Language :: Python :: 3.13', - - 'License :: OSI Approved :: MIT License', - - 'Intended Audience :: Developers', - 'Intended Audience :: Information Technology', - 'Intended Audience :: Science/Research', - - 'Topic :: Internet :: WWW/HTTP', - 'Topic :: System :: Archiving', - 'Topic :: System :: Archiving :: Backup', - 'Topic :: System :: Archiving :: Mirroring', - 'Topic :: Utilities', + +def _load_requirements(group): + with PIPFILE.open('rb') as fh: + dependencies = tomllib.load(fh) + + return [ + package if specifier == '*' else f'{package}{specifier}' + for package, specifier in dependencies[group].items() ] -) + + +def get_setup_kwargs(): + return { + 'name': 'ipwb', + 'version': __version__, + 'url': 'https://github.com/oduwsdl/ipwb', + 'download_url': "https://github.com/oduwsdl/ipwb", + 'author': 'Mat Kelly', + 'author_email': 'me@matkelly.com', + 'description': desc, + 'packages': ['ipwb'], + 'python_requires': '>=3.9', + 'license': 'MIT', + 'long_description': README.read_text(), + 'long_description_content_type': "text/markdown", + 'provides': [ + 'ipwb' + ], + 'install_requires': _load_requirements('packages'), + 'tests_require': _load_requirements('dev-packages'), + 'entry_points': """ + [console_scripts] + ipwb = ipwb.__main__:main + """, + 'package_data': { + 'ipwb': [ + 'assets/*.*', + 'assets/favicons/*.*', + 'templates/*.*' + ] + }, + 'zip_safe': False, + 'keywords': 'http web archives ipfs distributed odu wayback memento', + 'classifiers': [ + 'Development Status :: 4 - Beta', + + 'Environment :: Web Environment', + + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.13', + + 'License :: OSI Approved :: MIT License', + + 'Intended Audience :: Developers', + 'Intended Audience :: Information Technology', + 'Intended Audience :: Science/Research', + + 'Topic :: Internet :: WWW/HTTP', + 'Topic :: System :: Archiving', + 'Topic :: System :: Archiving :: Backup', + 'Topic :: System :: Archiving :: Mirroring', + 'Topic :: Utilities', + ] + } + + +if __name__ == '__main__': + setup(**get_setup_kwargs()) # Publish to pypi: # rm -rf dist; python setup.py sdist bdist_wheel; twine upload dist/* diff --git a/test-requirements.txt b/test-requirements.txt deleted file mode 100644 index d0f1b941..00000000 --- a/test-requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -flake8>=3.7.9 -pytest>=5.3.5 -pytest-cov -pytest-flake8 -setuptools diff --git a/tests/test_setup.py b/tests/test_setup.py new file mode 100644 index 00000000..8ccab3a7 --- /dev/null +++ b/tests/test_setup.py @@ -0,0 +1,40 @@ +import importlib.util +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] + + +def _load_setup_module(): + spec = importlib.util.spec_from_file_location('setup_module', ROOT / 'setup.py') + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_setup_uses_pipfile_runtime_dependencies(): + setup_module = _load_setup_module() + + assert setup_module.get_setup_kwargs()['install_requires'] == [ + 'warcio>=1.5.3', + 'ipfshttpclient>=0.8.0a', + 'Flask>=3.0', + 'pycryptodome>=3.4.11', + 'requests>=2.19.1', + 'beautifulsoup4>=4.6.3', + 'surt>=0.3.0', + 'multiaddr>=0.0.9', + 'packaging==23.0' + ] + + +def test_setup_uses_pipfile_dev_dependencies(): + setup_module = _load_setup_module() + + assert setup_module.get_setup_kwargs()['tests_require'] == [ + 'flake8>=3.7.9', + 'pycodestyle', + 'pytest>=5.3.5,<9', + 'pytest-cov', + 'pytest-flake8' + ] From 84cc785f41562dbe725f7f51c732382c9f6f8591 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 15 Jun 2026 17:40:29 +0000 Subject: [PATCH 3/4] Harden Pipfile packaging setup --- .github/workflows/comment-run.yml | 16 +--------------- .github/workflows/test.yml | 15 +-------------- Pipfile | 1 + pyproject.toml | 6 ++++++ setup.py | 20 +++++++++++++------- tests/test_setup.py | 27 +++++++++++++-------------- 6 files changed, 35 insertions(+), 50 deletions(-) create mode 100644 pyproject.toml diff --git a/.github/workflows/comment-run.yml b/.github/workflows/comment-run.yml index ed6b4034..b257c3b2 100644 --- a/.github/workflows/comment-run.yml +++ b/.github/workflows/comment-run.yml @@ -24,21 +24,7 @@ jobs: - name: Install Test Dependencies run: | pip install --upgrade pip - python - <<'PY' - import subprocess - import sys - from setup import get_setup_kwargs - - subprocess.check_call([ - sys.executable, - '-m', - 'pip', - 'install', - *get_setup_kwargs()['tests_require'], - ]) - PY - - name: Install IPWB from Source - run: pip install . + pip install '.[test]' - name: Execute Code in Comment uses: ibnesayeed/actions-comment-run@master with: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 38600bfc..5b52e66b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -42,20 +42,7 @@ jobs: shell: bash run: | pip install --upgrade pip - pip install . - python - <<'PY' - import subprocess - import sys - from setup import get_setup_kwargs - - subprocess.check_call([ - sys.executable, - '-m', - 'pip', - 'install', - *get_setup_kwargs()['tests_require'], - ]) - PY + pip install '.[test]' - name: Run Tests shell: bash run: py.test -s --cov=./ diff --git a/Pipfile b/Pipfile index 3d6255dd..12d0ff7a 100644 --- a/Pipfile +++ b/Pipfile @@ -20,3 +20,4 @@ pycodestyle = "*" pytest = ">=5.3.5,<9" pytest-cov = "*" pytest-flake8 = "*" +tomli = ">=2.0.1" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..7b895b01 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,6 @@ +[build-system] +requires = [ + "setuptools>=78.1.1", + "tomli>=2.0.1; python_version < '3.11'", +] +build-backend = "setuptools.build_meta" diff --git a/setup.py b/setup.py index 60b6d50b..568abdb9 100644 --- a/setup.py +++ b/setup.py @@ -4,22 +4,25 @@ from setuptools import setup -from ipwb import __version__ - try: import tomllib except ModuleNotFoundError: - try: - import tomli as tomllib - except ModuleNotFoundError: - from pip._vendor import tomli as tomllib + import tomli as tomllib ROOT = Path(__file__).resolve().parent +PACKAGE_INIT = ROOT / 'ipwb' / '__init__.py' PIPFILE = ROOT / 'Pipfile' README = ROOT / 'README.md' desc = """InterPlanetary Wayback (ipwb): Web Archive integration with IPFS""" +def _load_version(): + for line in PACKAGE_INIT.read_text().splitlines(): + if line.startswith('__version__ = '): + return line.split('=', 1)[1].strip().strip("'\"") + raise RuntimeError('Unable to determine package version.') + + def _load_requirements(group): with PIPFILE.open('rb') as fh: dependencies = tomllib.load(fh) @@ -33,7 +36,7 @@ def _load_requirements(group): def get_setup_kwargs(): return { 'name': 'ipwb', - 'version': __version__, + 'version': _load_version(), 'url': 'https://github.com/oduwsdl/ipwb', 'download_url': "https://github.com/oduwsdl/ipwb", 'author': 'Mat Kelly', @@ -49,6 +52,9 @@ def get_setup_kwargs(): ], 'install_requires': _load_requirements('packages'), 'tests_require': _load_requirements('dev-packages'), + 'extras_require': { + 'test': _load_requirements('dev-packages') + }, 'entry_points': """ [console_scripts] ipwb = ipwb.__main__:main diff --git a/tests/test_setup.py b/tests/test_setup.py index 8ccab3a7..022b888f 100644 --- a/tests/test_setup.py +++ b/tests/test_setup.py @@ -1,6 +1,11 @@ import importlib.util from pathlib import Path +try: + import tomllib +except ModuleNotFoundError: + import tomli as tomllib + ROOT = Path(__file__).resolve().parents[1] @@ -14,27 +19,21 @@ def _load_setup_module(): def test_setup_uses_pipfile_runtime_dependencies(): setup_module = _load_setup_module() + with (ROOT / 'Pipfile').open('rb') as fh: + pipfile = tomllib.load(fh) assert setup_module.get_setup_kwargs()['install_requires'] == [ - 'warcio>=1.5.3', - 'ipfshttpclient>=0.8.0a', - 'Flask>=3.0', - 'pycryptodome>=3.4.11', - 'requests>=2.19.1', - 'beautifulsoup4>=4.6.3', - 'surt>=0.3.0', - 'multiaddr>=0.0.9', - 'packaging==23.0' + package if specifier == '*' else f'{package}{specifier}' + for package, specifier in pipfile['packages'].items() ] def test_setup_uses_pipfile_dev_dependencies(): setup_module = _load_setup_module() + with (ROOT / 'Pipfile').open('rb') as fh: + pipfile = tomllib.load(fh) assert setup_module.get_setup_kwargs()['tests_require'] == [ - 'flake8>=3.7.9', - 'pycodestyle', - 'pytest>=5.3.5,<9', - 'pytest-cov', - 'pytest-flake8' + package if specifier == '*' else f'{package}{specifier}' + for package, specifier in pipfile['dev-packages'].items() ] From 00e43c09211ef451cfbbf13ccc37fe7e5b50e9ba Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 15 Jun 2026 17:41:30 +0000 Subject: [PATCH 4/4] Cache Pipfile metadata reads --- setup.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 568abdb9..b4f40283 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,6 @@ #!/usr/bin/env python +from functools import lru_cache from pathlib import Path from setuptools import setup @@ -23,13 +24,16 @@ def _load_version(): raise RuntimeError('Unable to determine package version.') -def _load_requirements(group): +@lru_cache(maxsize=1) +def _load_pipfile(): with PIPFILE.open('rb') as fh: - dependencies = tomllib.load(fh) + return tomllib.load(fh) + +def _load_requirements(group): return [ package if specifier == '*' else f'{package}{specifier}' - for package, specifier in dependencies[group].items() + for package, specifier in _load_pipfile()[group].items() ]