From a5188910294c8ea217b98b5f2955619f85cae5aa Mon Sep 17 00:00:00 2001 From: femalves Date: Wed, 8 Apr 2026 15:57:55 -0400 Subject: [PATCH 1/3] Upgrade biblib to Python 3.12 --- .github/workflows/python_actions.yml | 4 ++-- Dockerfile | 21 +++++++++++++++++++++ alembic/env.py | 1 - biblib/manage.py | 3 ++- biblib/models.py | 16 ++++++++-------- biblib/tests/unit_tests/test_manage.py | 19 ++++++++++--------- config.py | 1 + pyproject.toml | 19 ++++++++++--------- 8 files changed, 54 insertions(+), 30 deletions(-) create mode 100644 Dockerfile create mode 100644 config.py diff --git a/.github/workflows/python_actions.yml b/.github/workflows/python_actions.yml index e54b4d8..9a1c841 100644 --- a/.github/workflows/python_actions.yml +++ b/.github/workflows/python_actions.yml @@ -11,11 +11,11 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: '3.10' + python-version: '3.12' - name: Install dependencies run: | - python -m pip install "pip==24" setuptools==57.5.0 wheel + python -m pip install "pip==24" "setuptools>=62.0.0,<70.0.0" wheel pip install ".[dev]" - name: Test with pytest diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..20971ee --- /dev/null +++ b/Dockerfile @@ -0,0 +1,21 @@ +FROM python:3.12.8 + +# Set the working directory in the container to /app +WORKDIR /app + +ENV UBUNTU_FRONTEND=noninteractive +USER root +RUN apt-get update && apt-get install -y postgresql + +# Add the current directory contents into the container at /app +ADD . /app + +# setuptools>=58 breaks support for use_2to3 that is used by ConcurrentLogHandler in adsmutils +RUN pip install "pip==24" "setuptools>=62.0.0,<70.0.0" + +# Install dependencies +RUN pip install . + +RUN useradd -ms /bin/bash ads + +EXPOSE 5000 diff --git a/alembic/env.py b/alembic/env.py index 9411ddc..9e9cc5a 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -1,4 +1,3 @@ -from __future__ import with_statement from alembic import context from sqlalchemy import engine_from_config, pool from logging.config import fileConfig diff --git a/biblib/manage.py b/biblib/manage.py index 206c629..53e2f98 100644 --- a/biblib/manage.py +++ b/biblib/manage.py @@ -6,6 +6,7 @@ from flask import current_app from datetime import datetime from dateutil.relativedelta import relativedelta +from sqlalchemy import text import sqlalchemy_continuum class DeleteStaleUsers: @@ -19,7 +20,7 @@ def run(self, app=None): with app.app_context(): with current_app.session_scope() as session: # Obtain the list of API users - postgres_search_text = 'SELECT id FROM users;' + postgres_search_text = text('SELECT id FROM users;') result = session.execute(postgres_search_text).fetchall() list_of_api_users = [int(r[0]) for r in result] diff --git a/biblib/models.py b/biblib/models.py index ec0633f..0f4d3bf 100644 --- a/biblib/models.py +++ b/biblib/models.py @@ -6,11 +6,11 @@ """ import uuid -from datetime import datetime +from datetime import datetime, timezone from sqlalchemy.dialects.postgresql import UUID, JSON from sqlalchemy.ext.mutable import Mutable from sqlalchemy.types import TypeDecorator, CHAR, String as StringType -from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import declarative_base from sqlalchemy import Column, Integer, String, ForeignKey, DateTime, Boolean, UnicodeText, UniqueConstraint from sqlalchemy.orm import relationship, configure_mappers from sqlalchemy_continuum import make_versioned @@ -189,13 +189,13 @@ class Notes(Base): date_created = Column( DateTime, nullable=False, - default=datetime.utcnow + default=lambda: datetime.now(timezone.utc).replace(tzinfo=None) ) date_last_modified = Column( DateTime, nullable=False, - default=datetime.utcnow, - onupdate=datetime.utcnow + default=lambda: datetime.now(timezone.utc).replace(tzinfo=None), + onupdate=lambda: datetime.now(timezone.utc).replace(tzinfo=None) ) def __repr__(self): @@ -251,13 +251,13 @@ class Library(Base): date_created = Column( DateTime, nullable=False, - default=datetime.utcnow + default=lambda: datetime.now(timezone.utc).replace(tzinfo=None) ) date_last_modified = Column( DateTime, nullable=False, - default=datetime.utcnow, - onupdate=datetime.utcnow + default=lambda: datetime.now(timezone.utc).replace(tzinfo=None), + onupdate=lambda: datetime.now(timezone.utc).replace(tzinfo=None) ) permissions = relationship('Permissions', backref='library', diff --git a/biblib/tests/unit_tests/test_manage.py b/biblib/tests/unit_tests/test_manage.py index c47a00f..f5219ef 100644 --- a/biblib/tests/unit_tests/test_manage.py +++ b/biblib/tests/unit_tests/test_manage.py @@ -6,6 +6,7 @@ from biblib.manage import DeleteObsoleteVersionsNumber, DeleteStaleUsers, DeleteObsoleteVersionsTime from biblib.models import User, Library, Permissions, Notes from sqlalchemy.orm.exc import NoResultFound +from sqlalchemy import text from biblib.tests.base import TestCaseDatabase import sqlalchemy_continuum import freezegun @@ -53,8 +54,8 @@ def test_delete_stale_users(self): with self.app.session_scope() as session: # We do not add user 1 to the API database - session.execute('create table users (id integer, random integer);') - session.execute('insert into users (id, random) values (2, 7);') + session.execute(text('create table users (id integer, random integer);')) + session.execute(text('insert into users (id, random) values (2, 7);')) session.commit() with self.app.session_scope() as session: @@ -171,7 +172,7 @@ def test_delete_stale_users(self): raise finally: # Destroy the tables - session.execute('drop table users;') + session.execute(text('drop table users;')) pass def test_delete_obsolete_versions_number(self): @@ -184,8 +185,8 @@ def test_delete_obsolete_versions_number(self): with self.app.session_scope() as session: # We do not add user 1 to the API database - session.execute('create table users (id integer, random integer);') - session.execute('insert into users (id, random) values (2, 7);') + session.execute(text('create table users (id integer, random integer);')) + session.execute(text('insert into users (id, random) values (2, 7);')) session.commit() with self.app.session_scope() as session: @@ -340,7 +341,7 @@ def test_delete_obsolete_versions_number(self): raise finally: # Destroy the tables - session.execute('drop table users;') + session.execute(text('drop table users;')) pass def test_delete_obsolete_versions_time(self): @@ -353,8 +354,8 @@ def test_delete_obsolete_versions_time(self): with self.app.session_scope() as session: # We do not add user 1 to the API database - session.execute('create table users (id integer, random integer);') - session.execute('insert into users (id, random) values (2, 7);') + session.execute(text('create table users (id integer, random integer);')) + session.execute(text('insert into users (id, random) values (2, 7);')) session.commit() with self.app.session_scope() as session: @@ -535,7 +536,7 @@ def test_delete_obsolete_versions_time(self): raise finally: # Destroy the tables - session.execute('drop table users;') + session.execute(text('drop table users;')) pass if __name__ == '__main__': diff --git a/config.py b/config.py new file mode 100644 index 0000000..4a650ed --- /dev/null +++ b/config.py @@ -0,0 +1 @@ +from biblib.config import * diff --git a/pyproject.toml b/pyproject.toml index 33e898d..24ceef9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,10 +7,10 @@ license = { text = "MIT" } readme = "README.md" packages = ["biblib"] dependencies = [ - "adsmutils @ git+https://github.com/adsabs/ADSMicroserviceUtils.git@v1.3.0", + "adsmutils @ git+https://github.com/femalves/ADSMicroserviceUtils.git@1f2c55b", "alembic==1.12.0", "psycopg2-binary==2.9.9", - "sqlalchemy-continuum==1.3.6", + "sqlalchemy-continuum==1.3.15", "Flask-Mail==0.9.1", "Flask-Email==1.4.4", "Jinja2==3.1.2", @@ -24,22 +24,23 @@ dev = [ "Flask-Testing==0.8.1", "httpretty==1.1.4", "testing.postgresql==1.3.0", - "pytest==6.2.5", - "pytest-cov==3.0.0", + "pytest==7.4.4", + "pytest-cov==5.0.0", "Faker==22.0.0", "factory-boy==3.3.0", "freezegun==1.4.0", "httmock==1.4.0", "mock==4.0.3", - "flake8==4.0.1", - "black==22.3.0", + "flake8==7.0.0", + "black==24.3.0", "isort==5.12.0", - "coveralls==3.3.1" + "coverage==7.6.1", + "coveralls==4.0.1" ] [build-system] requires = [ - "setuptools==57.5.0", + "setuptools>=62.0.0,<70.0.0", "wheel", "flit_core >=3.2,<4", "ppsetuptools==2.0.2" @@ -52,7 +53,7 @@ testpaths = ["biblib/tests"] [tool.black] line-length = 88 -target-version = ['py310'] +target-version = ['py312'] [tool.isort] profile = "black" From 1f3ee68554dc7de7e0407a249c9d27b1c35f82ed Mon Sep 17 00:00:00 2001 From: femalves Date: Tue, 21 Apr 2026 11:33:29 -0400 Subject: [PATCH 2/3] Migrate to Python 3.12, bump deps, and reorganize config --- .github/workflows/python_actions.yml | 1 - Dockerfile | 21 -------- README.md | 5 +- biblib/config.py | 49 ------------------ .../test_bb_and_classic_user_epic.py | 2 +- .../test_classic_user_epic.py | 2 +- config.py | 50 ++++++++++++++++++- pyproject.toml | 41 +++++++-------- 8 files changed, 74 insertions(+), 97 deletions(-) delete mode 100644 Dockerfile delete mode 100644 biblib/config.py diff --git a/.github/workflows/python_actions.yml b/.github/workflows/python_actions.yml index 9a1c841..2a8bc16 100644 --- a/.github/workflows/python_actions.yml +++ b/.github/workflows/python_actions.yml @@ -15,7 +15,6 @@ jobs: - name: Install dependencies run: | - python -m pip install "pip==24" "setuptools>=62.0.0,<70.0.0" wheel pip install ".[dev]" - name: Test with pytest diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 20971ee..0000000 --- a/Dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -FROM python:3.12.8 - -# Set the working directory in the container to /app -WORKDIR /app - -ENV UBUNTU_FRONTEND=noninteractive -USER root -RUN apt-get update && apt-get install -y postgresql - -# Add the current directory contents into the container at /app -ADD . /app - -# setuptools>=58 breaks support for use_2to3 that is used by ConcurrentLogHandler in adsmutils -RUN pip install "pip==24" "setuptools>=62.0.0,<70.0.0" - -# Install dependencies -RUN pip install . - -RUN useradd -ms /bin/bash ads - -EXPOSE 5000 diff --git a/README.md b/README.md index 843b45f..5a50158 100644 --- a/README.md +++ b/README.md @@ -27,8 +27,7 @@ docker run -d --name postgres -e POSTGRES_USER="postgres" -e POSTGRES_PASSWORD=" python3 -m venv python source python/bin/activate -# Install with legacy build support (requires pip 24 and specific setuptools) -python -m pip install "pip==24" setuptools==57.5.0 wheel +# Install the project and dev dependencies pip install -e ".[dev]" # Run tests @@ -48,7 +47,7 @@ All tests have been written top down, or in a Test-Driven Development approach, ### Running Biblib Locally To run a version of Biblib locally, a postgres database needs to be created and properly formatted for use with Biblib. This can be done with a local postgres instance or in a docker container using the following commands. -`local_config.py` should be created in `biblib/` and the environment variables must be adjusted to reflect the local environment. +`local_config.py` should be created at the repo root (next to `config.py`) and the environment variables must be adjusted to reflect the local environment. ```bash # Setup database diff --git a/biblib/config.py b/biblib/config.py deleted file mode 100644 index 544ecc5..0000000 --- a/biblib/config.py +++ /dev/null @@ -1,49 +0,0 @@ -# encoding: utf-8 -""" -Configuration file. Please prefix application specific config values with -the application name. -""" - -import os -import pwd - -LOG_PATH = os.path.abspath( - os.path.join(os.path.dirname(__file__), './') -) -LOG_PATH = '{home}/logs/'.format(home=LOG_PATH) -LOG_STDOUT = True - -if not os.path.isdir(LOG_PATH): - os.mkdir(LOG_PATH) - -# For running tests on TravisCI -SQLALCHEMY_DATABASE_URI = 'postgresql+psycopg2://postgres:postgres@localhost:5432/test_biblib' -SQLALCHEMY_BINDS = { - 'libraries': 'postgresql+psycopg2://postgres:postgres@localhost:5432/test_biblib' -} - -ENVIRONMENT = os.getenv('ENVIRONMENT', 'unset-env').lower() - -# These lines are necessary only if the app needs to be a client of the -# adsws-api -BIBLIB_TWOPOINTOH_SERVICE_URL = 'https://api.adsabs.edu/v1/harbour' -BIBLIB_CLASSIC_SERVICE_URL = 'https://api.adsabs.edu/v1/harbour' -BIBLIB_SOLR_BIG_QUERY_URL = 'https://api.adsabs.search/v1/bigquery' -BIBLIB_SOLR_SEARCH_URL = 'https://api.adsabs.harvard.edu/v1/search/query' -BIBLIB_USER_EMAIL_ADSWS_API_URL = 'https://api.adsabs.harvard.edu/v1/user' -BIBLIB_ADSWS_API_DB_URI = 'sqlite:////tmp/test.db' -BIBLIB_MAX_ROWS = 2000 -BIGQUERY_MAX_ROWS = 200 -BIBLIB_SOLR_BIG_QUERY_MIN = 10 - -MAIL_DEFAULT_SENDER = 'no-reply@adslabs.org' - -NUMBER_REVISIONS = 7 -REVISION_TIME = 1 - -# when set, the service will use it instead of the user's token -# (for requests where it makes sense - not all) -SERVICE_TOKEN = None - -# myADS token to allow general notifications that contain docs(library/) -READONLY_ALL_LIBRARIES_TOKEN = None diff --git a/biblib/tests/functional_tests/test_bb_and_classic_user_epic.py b/biblib/tests/functional_tests/test_bb_and_classic_user_epic.py index bb6cc74..68680ca 100644 --- a/biblib/tests/functional_tests/test_bb_and_classic_user_epic.py +++ b/biblib/tests/functional_tests/test_bb_and_classic_user_epic.py @@ -14,7 +14,7 @@ from biblib.tests.base import TestCaseDatabase, MockEmailService, \ MockSolrBigqueryService, MockEndPoint, MockClassicService from biblib.views.http_errors import NO_CLASSIC_ACCOUNT -from biblib.config import BIBLIB_CLASSIC_SERVICE_URL +from config import BIBLIB_CLASSIC_SERVICE_URL class TestBBClassicUserEpic(TestCaseDatabase): diff --git a/biblib/tests/functional_tests/test_classic_user_epic.py b/biblib/tests/functional_tests/test_classic_user_epic.py index 5b819ad..1690a99 100644 --- a/biblib/tests/functional_tests/test_classic_user_epic.py +++ b/biblib/tests/functional_tests/test_classic_user_epic.py @@ -14,7 +14,7 @@ from biblib.tests.base import TestCaseDatabase, MockEmailService, \ MockSolrBigqueryService, MockEndPoint, MockClassicService from biblib.views.http_errors import NO_CLASSIC_ACCOUNT -from biblib.config import BIBLIB_CLASSIC_SERVICE_URL +from config import BIBLIB_CLASSIC_SERVICE_URL @urlmatch(netloc=r'(.*\.)?{}.*'.format(BIBLIB_CLASSIC_SERVICE_URL)) diff --git a/config.py b/config.py index 4a650ed..544ecc5 100644 --- a/config.py +++ b/config.py @@ -1 +1,49 @@ -from biblib.config import * +# encoding: utf-8 +""" +Configuration file. Please prefix application specific config values with +the application name. +""" + +import os +import pwd + +LOG_PATH = os.path.abspath( + os.path.join(os.path.dirname(__file__), './') +) +LOG_PATH = '{home}/logs/'.format(home=LOG_PATH) +LOG_STDOUT = True + +if not os.path.isdir(LOG_PATH): + os.mkdir(LOG_PATH) + +# For running tests on TravisCI +SQLALCHEMY_DATABASE_URI = 'postgresql+psycopg2://postgres:postgres@localhost:5432/test_biblib' +SQLALCHEMY_BINDS = { + 'libraries': 'postgresql+psycopg2://postgres:postgres@localhost:5432/test_biblib' +} + +ENVIRONMENT = os.getenv('ENVIRONMENT', 'unset-env').lower() + +# These lines are necessary only if the app needs to be a client of the +# adsws-api +BIBLIB_TWOPOINTOH_SERVICE_URL = 'https://api.adsabs.edu/v1/harbour' +BIBLIB_CLASSIC_SERVICE_URL = 'https://api.adsabs.edu/v1/harbour' +BIBLIB_SOLR_BIG_QUERY_URL = 'https://api.adsabs.search/v1/bigquery' +BIBLIB_SOLR_SEARCH_URL = 'https://api.adsabs.harvard.edu/v1/search/query' +BIBLIB_USER_EMAIL_ADSWS_API_URL = 'https://api.adsabs.harvard.edu/v1/user' +BIBLIB_ADSWS_API_DB_URI = 'sqlite:////tmp/test.db' +BIBLIB_MAX_ROWS = 2000 +BIGQUERY_MAX_ROWS = 200 +BIBLIB_SOLR_BIG_QUERY_MIN = 10 + +MAIL_DEFAULT_SENDER = 'no-reply@adslabs.org' + +NUMBER_REVISIONS = 7 +REVISION_TIME = 1 + +# when set, the service will use it instead of the user's token +# (for requests where it makes sense - not all) +SERVICE_TOKEN = None + +# myADS token to allow general notifications that contain docs(library/) +READONLY_ALL_LIBRARIES_TOKEN = None diff --git a/pyproject.toml b/pyproject.toml index 24ceef9..0c87d11 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,15 +7,15 @@ license = { text = "MIT" } readme = "README.md" packages = ["biblib"] dependencies = [ - "adsmutils @ git+https://github.com/femalves/ADSMicroserviceUtils.git@1f2c55b", - "alembic==1.12.0", - "psycopg2-binary==2.9.9", - "sqlalchemy-continuum==1.3.15", - "Flask-Mail==0.9.1", + "adsmutils @ git+https://github.com/femalves/ADSMicroserviceUtils.git@de1108d", + "alembic==1.18.5", + "psycopg2-binary==2.9.12", + "sqlalchemy-continuum==1.6.0", + "Flask-Mail==0.10.0", "Flask-Email==1.4.4", - "Jinja2==3.1.2", - "markupsafe==2.1.3", - "itsdangerous==2.1.2", + "Jinja2==3.1.6", + "markupsafe==3.0.3", + "itsdangerous==2.2.0", "werkzeug==2.3.8" ] @@ -24,23 +24,23 @@ dev = [ "Flask-Testing==0.8.1", "httpretty==1.1.4", "testing.postgresql==1.3.0", - "pytest==7.4.4", - "pytest-cov==5.0.0", - "Faker==22.0.0", - "factory-boy==3.3.0", - "freezegun==1.4.0", + "pytest==9.1.1", + "pytest-cov==7.1.0", + "Faker==40.28.1", + "factory-boy==3.3.3", + "freezegun==1.5.5", "httmock==1.4.0", - "mock==4.0.3", - "flake8==7.0.0", - "black==24.3.0", - "isort==5.12.0", - "coverage==7.6.1", - "coveralls==4.0.1" + "mock==5.2.0", + "flake8==7.3.0", + "black==26.5.1", + "isort==8.0.1", + "coverage==7.15.0", + "coveralls==4.1.0" ] [build-system] requires = [ - "setuptools>=62.0.0,<70.0.0", + "setuptools>=62.0.0", "wheel", "flit_core >=3.2,<4", "ppsetuptools==2.0.2" @@ -50,6 +50,7 @@ build-backend = "flit_core.buildapi" [tool.pytest.ini_options] addopts = "--cov=biblib --cov-report=term-missing" testpaths = ["biblib/tests"] +pythonpath = ["."] [tool.black] line-length = 88 From 88263aa849d4446b1e42b18cf5c208625806fce0 Mon Sep 17 00:00:00 2001 From: femalves Date: Tue, 28 Jul 2026 11:57:35 -0400 Subject: [PATCH 3/3] Bump adsmutils pin to v2.0.0 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0c87d11..ee3a3d3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ license = { text = "MIT" } readme = "README.md" packages = ["biblib"] dependencies = [ - "adsmutils @ git+https://github.com/femalves/ADSMicroserviceUtils.git@de1108d", + "adsmutils @ git+https://github.com/adsabs/ADSMicroserviceUtils.git@v2.0.0", "alembic==1.18.5", "psycopg2-binary==2.9.12", "sqlalchemy-continuum==1.6.0",