diff --git a/bin/bibliotheca_circulation_update b/bin/bibliotheca_circulation_update index d75cb89707..fb7a06f69a 100755 --- a/bin/bibliotheca_circulation_update +++ b/bin/bibliotheca_circulation_update @@ -1,6 +1,6 @@ #!/usr/bin/env python """Manually kick off the Bibliotheca circulation update for one or all collections.""" -from palace.manager.integration.license.bibliotheca_scripts import ( +from palace.manager.integration.license.bibliotheca.scripts import ( CirculationUpdateCollection, ) diff --git a/bin/bibliotheca_import b/bin/bibliotheca_import index 48da797478..06ec659ab5 100755 --- a/bin/bibliotheca_import +++ b/bin/bibliotheca_import @@ -1,7 +1,7 @@ #!/usr/bin/env python """Manually kick off the Bibliotheca event import for one or all collections.""" -from palace.manager.integration.license.bibliotheca_scripts import ( +from palace.manager.integration.license.bibliotheca.scripts import ( ImportEventCollection, ) diff --git a/bin/bibliotheca_purchase_record_import b/bin/bibliotheca_purchase_record_import index d4f43faf42..ccbbd887c1 100755 --- a/bin/bibliotheca_purchase_record_import +++ b/bin/bibliotheca_purchase_record_import @@ -1,7 +1,7 @@ #!/usr/bin/env python """Manually kick off the Bibliotheca purchase record import for one or all collections.""" -from palace.manager.integration.license.bibliotheca_scripts import ( +from palace.manager.integration.license.bibliotheca.scripts import ( ImportPurchaseRecordCollection, ) diff --git a/bin/informational/bibliotheca-raw-bibliographic b/bin/informational/bibliotheca-raw-bibliographic index 87bc83f401..43d380f93f 100755 --- a/bin/informational/bibliotheca-raw-bibliographic +++ b/bin/informational/bibliotheca-raw-bibliographic @@ -1,7 +1,7 @@ #!/usr/bin/env python3 from xml.dom import minidom -from palace.manager.integration.license.bibliotheca import BibliothecaAPI +from palace.manager.integration.license.bibliotheca.api import BibliothecaAPI from palace.manager.scripts.input import IdentifierInputScript from palace.manager.sqlalchemy.model.collection import Collection diff --git a/bin/informational/bibliotheca-raw-patron-status b/bin/informational/bibliotheca-raw-patron-status index b532a308cd..620f083ab0 100755 --- a/bin/informational/bibliotheca-raw-patron-status +++ b/bin/informational/bibliotheca-raw-patron-status @@ -2,7 +2,7 @@ import sys from xml.dom import minidom -from palace.manager.integration.license.bibliotheca import BibliothecaAPI +from palace.manager.integration.license.bibliotheca.api import BibliothecaAPI from palace.manager.scripts.base import Script from palace.manager.sqlalchemy.model.collection import Collection from palace.manager.sqlalchemy.model.patron import Patron diff --git a/bin/repair/bibliotheca_bibliographic_coverage b/bin/repair/bibliotheca_bibliographic_coverage index b60d289b19..bed94e8849 100755 --- a/bin/repair/bibliotheca_bibliographic_coverage +++ b/bin/repair/bibliotheca_bibliographic_coverage @@ -2,7 +2,7 @@ """Make sure all Bibliotheca books have bibliographic coverage.""" -from palace.manager.integration.license.bibliotheca import ( +from palace.manager.integration.license.bibliotheca.coverage import ( BibliothecaBibliographicCoverageProvider, ) from palace.manager.scripts.coverage_provider import RunCollectionCoverageProviderScript diff --git a/src/palace/manager/celery/tasks/bibliotheca.py b/src/palace/manager/celery/tasks/bibliotheca.py index 7c8feadf48..02c8d2047c 100644 --- a/src/palace/manager/celery/tasks/bibliotheca.py +++ b/src/palace/manager/celery/tasks/bibliotheca.py @@ -32,15 +32,15 @@ from palace.manager.celery.importer import workflow_lock_guard from palace.manager.celery.task import Task from palace.manager.celery.utils import ModelNotFoundError, load_from_id, signature_with -from palace.manager.integration.license.bibliotheca import BibliothecaAPI -from palace.manager.integration.license.bibliotheca_circulation_updater import ( +from palace.manager.integration.license.bibliotheca.api import BibliothecaAPI +from palace.manager.integration.license.bibliotheca.circulation_updater import ( BibliothecaCirculationUpdater, ) -from palace.manager.integration.license.bibliotheca_importer import ( +from palace.manager.integration.license.bibliotheca.importer import ( EVENT_IMPORT_OVERLAP, BibliothecaEventImporter, ) -from palace.manager.integration.license.bibliotheca_purchase_record_importer import ( +from palace.manager.integration.license.bibliotheca.purchase_record_importer import ( DEFAULT_PURCHASE_RECORD_START_TIME, PURCHASE_RECORD_SERVICE_NAME, BibliothecaPurchaseRecordImporter, diff --git a/src/palace/manager/integration/license/bibliotheca.py b/src/palace/manager/integration/license/bibliotheca.py deleted file mode 100644 index 2603144781..0000000000 --- a/src/palace/manager/integration/license/bibliotheca.py +++ /dev/null @@ -1,1295 +0,0 @@ -from __future__ import annotations - -import hashlib -import hmac -import html -import itertools -import json -import logging -import re -import time -import urllib.parse -from abc import ABC -from collections.abc import Collection as CollectionT, Generator, Iterable -from datetime import datetime, timedelta -from io import BytesIO -from typing import Annotated, Any, Literal, Optional, Unpack, overload - -from flask_babel import lazy_gettext as _ -from frozendict import frozendict -from lxml.etree import Error, _Element -from pymarc import Record, parse_xml_to_array -from requests import Response -from sqlalchemy.orm import Session - -from palace.util.datetime_helpers import ( - strptime_utc, - to_utc, - utc_now, -) -from palace.util.log import LoggerMixin - -from palace.manager.api.circulation.base import ( - BaseCirculationAPI, - PatronActivityCirculationAPI, -) -from palace.manager.api.circulation.data import HoldInfo, LoanInfo -from palace.manager.api.circulation.exceptions import ( - AlreadyCheckedOut, - AlreadyOnHold, - CannotHold, - CannotLoan, - CannotReleaseHold, - CurrentlyAvailable, - NoAvailableCopies, - NoLicenses, - NotCheckedOut, - NotOnHold, - PatronHoldLimitReached, - PatronLoanLimitReached, - RemoteInitiatedServerError, -) -from palace.manager.api.circulation.fulfillment import DirectFulfillment -from palace.manager.api.circulation.settings import ( - BaseCirculationApiSettings, - BaseCirculationLoanSettings, -) -from palace.manager.api.selftest import HasCollectionSelfTests -from palace.manager.api.web_publication_manifest import FindawayManifest, SpineItem -from palace.manager.core.config import ( - CannotLoadConfiguration, - ConfigurationAttributeValue, -) -from palace.manager.core.coverage import BibliographicCoverageProvider, CoverageFailure -from palace.manager.core.selftest import SelfTestResult -from palace.manager.data_layer.bibliographic import BibliographicData -from palace.manager.data_layer.circulation import CirculationData -from palace.manager.data_layer.contributor import ContributorData -from palace.manager.data_layer.format import FormatData -from palace.manager.data_layer.identifier import IdentifierData -from palace.manager.data_layer.link import LinkData -from palace.manager.data_layer.measurement import MeasurementData -from palace.manager.data_layer.subject import SubjectData -from palace.manager.integration.settings import ( - FormFieldType, - FormMetadata, -) -from palace.manager.sqlalchemy.constants import DataSourceConstants -from palace.manager.sqlalchemy.model.circulationevent import CirculationEvent -from palace.manager.sqlalchemy.model.classification import Classification, Subject -from palace.manager.sqlalchemy.model.collection import Collection -from palace.manager.sqlalchemy.model.contributor import Contributor -from palace.manager.sqlalchemy.model.datasource import DataSource -from palace.manager.sqlalchemy.model.edition import Edition -from palace.manager.sqlalchemy.model.identifier import Identifier -from palace.manager.sqlalchemy.model.licensing import ( - DeliveryMechanism, - LicensePool, - LicensePoolDeliveryMechanism, - LicensePoolStatus, -) -from palace.manager.sqlalchemy.model.measurement import Measurement -from palace.manager.sqlalchemy.model.patron import Patron -from palace.manager.sqlalchemy.model.resource import Hyperlink, Representation -from palace.manager.util import base64 -from palace.manager.util.http.exception import RemoteIntegrationException -from palace.manager.util.http.http import HTTP, RequestKwargs -from palace.manager.util.problem_detail import BaseProblemDetailException -from palace.manager.util.xmlparser import XMLParser, XMLProcessor - - -class BibliothecaSettings(BaseCirculationApiSettings): - username: Annotated[ - str, - FormMetadata( - label=_("Account ID"), - required=True, - ), - ] - password: Annotated[ - str, - FormMetadata( - label=_("Account Key"), - required=True, - ), - ] - external_account_id: Annotated[ - str, - FormMetadata( - label=_("Library ID"), - required=True, - ), - ] - - -class BibliothecaLibrarySettings(BaseCirculationLoanSettings): - dont_display_reserves: Annotated[ - ConfigurationAttributeValue, - FormMetadata( - label=_("Show/Hide Titles with No Available Loans"), - required=False, - description=_( - "Titles with no available loans will not be displayed in the Catalog view." - ), - type=FormFieldType.SELECT, - options={ - ConfigurationAttributeValue.YESVALUE: "Show", - ConfigurationAttributeValue.NOVALUE: "Hide", - }, - ), - ] = ConfigurationAttributeValue.YESVALUE - - -class BibliothecaAPI( - PatronActivityCirculationAPI[BibliothecaSettings, BibliothecaLibrarySettings], - HasCollectionSelfTests, -): - AUTH_TIME_FORMAT = "%a, %d %b %Y %H:%M:%S GMT" - ARGUMENT_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S" - AUTHORIZATION_FORMAT = "3MCLAUTH %s:%s" - - DATETIME_HEADER = "3mcl-Datetime" - AUTHORIZATION_HEADER = "3mcl-Authorization" - VERSION_HEADER = "3mcl-Version" - - DEFAULT_VERSION = "2.0" - DEFAULT_BASE_URL = "https://partner.yourcloudlibrary.com/" - - MAX_AGE = timedelta(days=730).seconds - CAN_REVOKE_HOLD_WHEN_RESERVED = False - SET_DELIVERY_MECHANISM_AT = None - - SERVICE_NAME = "Bibliotheca" - - @classmethod - def settings_class(cls) -> type[BibliothecaSettings]: - return BibliothecaSettings - - @classmethod - def library_settings_class(cls) -> type[BibliothecaLibrarySettings]: - return BibliothecaLibrarySettings - - @classmethod - def label(cls) -> str: - return DataSourceConstants.BIBLIOTHECA - - @classmethod - def description(cls) -> str: - return "" - - def __init__(self, _db: Session, collection: Collection) -> None: - super().__init__(_db, collection) - - self._db = _db - self.version = self.DEFAULT_VERSION - self.account_id = self.settings.username - self.account_key = self.settings.password - self.library_id = self.settings.external_account_id - self.base_url = self.DEFAULT_BASE_URL - - if not self.account_id or not self.account_key or not self.library_id: - raise CannotLoadConfiguration("Bibliotheca configuration is incomplete.") - - self.item_list_parser = ItemListParser() - self.collection_id = collection.id - - @property - def data_source(self) -> DataSource: - return DataSource.lookup(self._db, DataSource.BIBLIOTHECA, autocreate=True) - - def now(self) -> str: - """Return the current GMT time in the format 3M expects.""" - return time.strftime(self.AUTH_TIME_FORMAT, time.gmtime()) - - def sign(self, method: str, headers: dict[str, str], path: str) -> None: - """Add appropriate headers to a request.""" - authorization, now = self.authorization(method, path) - headers[self.DATETIME_HEADER] = now - headers[self.VERSION_HEADER] = self.version - headers[self.AUTHORIZATION_HEADER] = authorization - - def authorization(self, method: str, path: str) -> tuple[str, str]: - signature, now = self.signature(method, path) - auth = self.AUTHORIZATION_FORMAT % (self.account_id, signature) - return auth, now - - def signature(self, method: str, path: str) -> tuple[str, str]: - now = self.now() - signature_components = [now, method, path] - signature_string = "\n".join(signature_components) - digest = hmac.new( - self.account_key.encode("utf-8"), - msg=signature_string.encode("utf-8"), - digestmod=hashlib.sha256, - ).digest() - signature = base64.standard_b64encode(digest) - return signature, now - - def full_url(self, path: str) -> str: - if not path.startswith("/cirrus"): - path = self.full_path(path) - return urllib.parse.urljoin(self.base_url, path) - - def full_path(self, path: str) -> str: - if not path.startswith("/"): - path = "/" + path - if not path.startswith("/cirrus"): - path = f"/cirrus/library/{self.library_id}{path}" - return path - - def request( - self, - path: str, - body: str | None = None, - method: str = "GET", - ) -> Response: - path = self.full_path(path) - url = self.full_url(path) - if method == "GET": - headers = {"Accept": "application/xml"} - else: - headers = {"Content-Type": "application/xml"} - self.sign(method, headers, path) - return self._request_with_timeout( - method, - url, - data=body, - headers=headers, - allow_redirects=False, - timeout=60, - ) - - def marc_request( - self, start: datetime, end: datetime, offset: int = 1, limit: int = 50 - ) -> Generator[Record]: - """Make an HTTP request to look up the MARC records for books purchased - between two given dates. - - :param start: A datetime to start looking for purchases. - :param end: A datetime to stop looking for purchases. - :param offset: An offset used to paginate results. - :param limit: A limit used to paginate results. - :raise: An appropriate exception if the request returned a non-200 - status code. An empty response body is not an error: it is treated - as "no records" and the generator simply yields nothing. - :yield: A list of MARC records. - """ - start_param = start.strftime(self.ARGUMENT_TIME_FORMAT) - end_param = end.strftime(self.ARGUMENT_TIME_FORMAT) - url = "data/marc?startdate=%s&enddate=%s&offset=%d&limit=%d" % ( - start_param, - end_param, - offset, - limit, - ) - response = self.request(url) - if response.status_code != 200: - raise ErrorParser().process_first(response.content) - if not response.content.strip(): - # Bibliotheca sometimes returns an empty body (HTTP 200 with no - # XML document) for a window that contains no purchase records. - # pymarc's parse_xml_to_array raises SAXException("no element - # found") on an empty document, so treat an empty body as "no - # records" and yield nothing rather than letting it propagate. - self.log.info( - f"Bibliotheca MARC request to '{url}' returned an empty " - "response body; treating as no records." - ) - return - yield from parse_xml_to_array(BytesIO(response.content)) - - def bibliographic_lookup_request(self, identifiers: CollectionT[str]) -> bytes: - """Make an HTTP request to look up current bibliographic and - circulation information for the given `identifiers`. - - :param identifiers: Strings containing Bibliotheca identifiers. - :return: A string containing an XML document, or None if there was - an error not handled as an exception. - """ - url = "/items/" + ",".join(identifiers) - response = self.request(url) - return response.content - - def bibliographic_lookup( - self, identifiers: CollectionT[str | Identifier] | str | Identifier - ) -> list[BibliographicData]: - """Look up current bibliographic and circulation information for the - given `identifiers`. - - :param identifiers: A list containing either Identifier - objects or Bibliotheca identifier strings. - :raise RemoteInitiatedServerError: If Bibliotheca returns an empty - response body (HTTP 200 with no XML document). See the comment - below for why this is treated as a transient error rather than as - "no items found". - """ - identifiers_list = ( - [identifiers] - if isinstance(identifiers, Identifier) or isinstance(identifiers, str) - else identifiers - ) - identifier_strings = [] - for i in identifiers_list: - if isinstance(i, Identifier): - i = i.identifier - identifier_strings.append(i) - - data = self.bibliographic_lookup_request(identifier_strings) - if not data.strip(): - # Bibliotheca occasionally returns an empty body (HTTP 200 with no - # XML document). An empty document cannot be parsed as XML (lxml - # raises "Document is empty" even in recovery mode). Unlike a - # well-formed document that simply omits some of the requested - # items, an empty body tells us nothing about which titles still - # exist, so we must not treat it as "no items returned": that would - # make BibliothecaCirculationUpdater._process_batch zero out the - # availability of every requested identifier as if it had been - # removed from circulation. Instead, raise a transient remote error - # -- mirroring how marc_request/ErrorParser treat empty or malformed - # Bibliotheca responses -- so the caller retries rather than - # corrupting availability data. - raise RemoteInitiatedServerError( - "Bibliotheca returned an empty response body for a bibliographic lookup.", - self.SERVICE_NAME, - ) - return [ - bibliographic for bibliographic in self.item_list_parser.process_all(data) - ] - - def _request_with_timeout( - self, http_method: str, url: str, **kwargs: Unpack[RequestKwargs] - ) -> Response: - """This will be overridden in MockBibliothecaAPI.""" - return HTTP.request_with_timeout(http_method, url, **kwargs) - - def _run_self_tests(self, _db: Session) -> Generator[SelfTestResult]: - def _count_events() -> str: - now = utc_now() - five_minutes_ago = now - timedelta(minutes=5) - count = len(list(self.get_events_between(five_minutes_ago, now))) - return "Found %d event(s)" % count - - yield self.run_test( - "Asking for circulation events for the last five minutes", _count_events - ) - - for result in self.default_patrons(self.collection): - if isinstance(result, SelfTestResult): - yield result - continue - library, patron, pin = result - - def _count_activity() -> str: - result = self.patron_activity(patron, pin) - return "Found %d loans/holds" % len(list(result)) - - yield self.run_test( - "Checking activity for test patron for library %s" % library.name, - _count_activity, - ) - - def get_events_between( - self, start: datetime, end: datetime, no_events_error: bool = False - ) -> Generator[tuple[str, str, str | None, datetime, datetime | None, str]]: - """Return event objects for events between the given times.""" - start_str = start.strftime(self.ARGUMENT_TIME_FORMAT) - end_str = end.strftime(self.ARGUMENT_TIME_FORMAT) - url = f"data/cloudevents?startdate={start_str}&enddate={end_str}" - response = self.request(url) - try: - events = EventParser().process_all(response.content, no_events_error) - except Exception as e: - self.log.error( - "Error parsing Bibliotheca response content: %s", - response.content, - exc_info=e, - ) - raise e - return events - - def update_availability(self, licensepool: LicensePool) -> None: - """Update the availability information for a single LicensePool.""" - # Local import to avoid circular dependency between bibliotheca.py and - # bibliotheca_circulation_updater.py (the updater imports BibliothecaAPI). - from palace.manager.integration.license.bibliotheca_circulation_updater import ( - BibliothecaCirculationUpdater, - ) - - updater = BibliothecaCirculationUpdater( - self._db, licensepool.collection, api=self - ) - updater.process_identifiers([licensepool.identifier]) - - def _patron_activity_request(self, patron: Patron) -> Response: - patron_id = patron.authorization_identifier - path = "circulation/patron/%s" % patron_id - return self.request(path) - - def patron_activity( - self, patron: Patron, pin: str | None - ) -> Iterable[LoanInfo | HoldInfo]: - response = self._patron_activity_request(patron) - try: - return PatronCirculationParser(self.collection).process_all( - response.content - ) - except Error as e: - # XML parse error from remote. - raise RemoteIntegrationException( - response.url, "Unable to parse response XML." - ) from e - - TEMPLATE = "<%(request_type)s>%(item_id)s%(patron_id)s" - - def checkout( - self, - patron_obj: Patron, - patron_password: str | None, - licensepool: LicensePool, - delivery_mechanism: LicensePoolDeliveryMechanism | None, - ) -> LoanInfo: - """Check out a book on behalf of a patron. - - :param patron_obj: a Patron object for the patron who wants - to check out the book. - - :param patron_password: The patron's alleged password. Not used here - since Bibliotheca trusts Simplified to do the check ahead of time. - - :param licensepool: LicensePool for the book to be checked out. - - :return: a LoanInfo object - """ - bibliotheca_id = licensepool.identifier.identifier - patron_identifier = patron_obj.authorization_identifier - args = dict( - request_type="CheckoutRequest", - item_id=bibliotheca_id, - patron_id=patron_identifier, - ) - body = self.TEMPLATE % args - response = self.request("checkout", body, method="PUT") - if response.status_code == 201: - # New loan - start_date = utc_now() - elif response.status_code == 200: - # Old loan -- we don't know the start date - start_date = None - else: - # Error condition. - error = ErrorParser().process_first(response.content) - if isinstance(error, AlreadyCheckedOut): - # It's already checked out. No problem. - pass - else: - raise error - - # At this point we know we have a loan. - loan_expires = CheckoutResponseParser().process_first(response.content) - loan = LoanInfo.from_license_pool( - licensepool, - end_date=loan_expires, - ) - return loan - - def fulfill( - self, - patron: Patron, - password: str, - pool: LicensePool, - delivery_mechanism: LicensePoolDeliveryMechanism, - **kwargs: Unpack[BaseCirculationAPI.FulfillKwargs], - ) -> DirectFulfillment: - """Get the actual resource file to the patron.""" - if ( - delivery_mechanism.delivery_mechanism.drm_scheme - == DeliveryMechanism.FINDAWAY_DRM - ): - fulfill_method = self.get_audio_fulfillment_file - content_transformation = self.findaway_license_to_webpub_manifest - else: - fulfill_method = self.get_fulfillment_file - content_transformation = None - response = fulfill_method( - patron.authorization_identifier, pool.identifier.identifier - ) - content: str | bytes = response.content - content_type = None - if content_transformation: - try: - content_type, content = content_transformation(pool, content) - except Exception as e: - self.log.error( - "Error transforming fulfillment document: %s", - response.content, - exc_info=e, - ) - return DirectFulfillment( - content=content, - content_type=content_type or response.headers.get("Content-Type"), - ) - - def get_fulfillment_file( - self, patron_id: str | None, bibliotheca_id: str - ) -> Response: - args = dict( - request_type="ACSMRequest", item_id=bibliotheca_id, patron_id=patron_id - ) - body = self.TEMPLATE % args - return self.request("GetItemACSM", body, method="PUT") - - def get_audio_fulfillment_file( - self, patron_id: str | None, bibliotheca_id: str - ) -> Response: - args = dict( - request_type="AudioFulfillmentRequest", - item_id=bibliotheca_id, - patron_id=patron_id, - ) - body = self.TEMPLATE % args - return self.request("GetItemAudioFulfillment", body, method="POST") - - def checkin(self, patron: Patron, pin: str, licensepool: LicensePool) -> None: - patron_id = patron.authorization_identifier - item_id = licensepool.identifier.identifier - args = dict(request_type="CheckinRequest", item_id=item_id, patron_id=patron_id) - body = self.TEMPLATE % args - self.request("checkin", body, method="PUT") - - def place_hold( - self, - patron: Patron, - pin: str | None, - licensepool: LicensePool, - notification_email_address: str | None = None, - ) -> HoldInfo: - """Place a hold. - - :return: a HoldInfo object. - """ - patron_id = patron.authorization_identifier - item_id = licensepool.identifier.identifier - args = dict( - request_type="PlaceHoldRequest", item_id=item_id, patron_id=patron_id - ) - body = self.TEMPLATE % args - response = self.request("placehold", body, method="PUT") - # The response comes in as a byte string that we must - # convert into a string. - response_content = None - if response.content: - response_content = response.content.decode("utf-8") - if response.status_code in (200, 201): - start_date = utc_now() - end_date = HoldResponseParser().process_first(response_content) - return HoldInfo.from_license_pool( - licensepool, - start_date=start_date, - end_date=end_date, - hold_position=None, - ) - else: - if not response_content: - raise CannotHold() - error = ErrorParser().process_first(response_content) - if isinstance(error, Exception): - raise error - else: - raise CannotHold(error) - - def release_hold(self, patron: Patron, pin: str, licensepool: LicensePool) -> None: - patron_id = patron.authorization_identifier - item_id = licensepool.identifier.identifier - args = dict( - request_type="CancelHoldRequest", item_id=item_id, patron_id=patron_id - ) - body = self.TEMPLATE % args - response = self.request("cancelhold", body, method="PUT") - if response.status_code not in (200, 404): - raise CannotReleaseHold() - - @classmethod - def findaway_license_to_webpub_manifest( - cls, license_pool: LicensePool, findaway_license: str | bytes | dict[str, Any] - ) -> tuple[str, str]: - """Convert a Bibliotheca license document to a FindawayManifest - suitable for serving to a mobile client. - - :param license_pool: A LicensePool for the title in question. - This will be used to fill in basic bibliographic information. - - :param findaway_license: A string containing a Findaway - license document via Bibliotheca, or a dictionary - representing such a document loaded into JSON form. - """ - if isinstance(findaway_license, (bytes, str)): - findaway_license = json.loads(findaway_license) - assert isinstance( - findaway_license, dict - ), "Expected a JSON object, got %s" % type(findaway_license) - - kwargs = {} - for findaway_extension in [ - "accountId", - "checkoutId", - "fulfillmentId", - "licenseId", - "sessionKey", - ]: - value = findaway_license.get(findaway_extension, None) - kwargs[findaway_extension] = value - - # Create the SpineItem objects. - audio_format = findaway_license.get("format") - if audio_format == "MP3": - part_media_type = Representation.MP3_MEDIA_TYPE - else: - cls.logger().error( - "Unknown Findaway audio format encountered: %s", audio_format - ) - part_media_type = None - - spine_items = [] - for part in findaway_license["items"]: - title = part.get("title") - - # TODO: Incoming duration appears to be measured in - # milliseconds. This assumption makes our example - # audiobook take about 7.9 hours, and no other reasonable - # assumption is in the right order of magnitude. But this - # needs to be explicitly verified. - duration = part.get("duration", 0) / 1000.0 - - part_number = int(part.get("part", 0)) - - sequence = int(part.get("sequence", 0)) - - spine_items.append(SpineItem(title, duration, part_number, sequence)) - - # Create a FindawayManifest object and then convert it - # to a string. - manifest = FindawayManifest( - license_pool=license_pool, spine_items=spine_items, **kwargs - ) - - return DeliveryMechanism.FINDAWAY_DRM, str(manifest) - - -class ItemListParser(XMLProcessor[BibliographicData], LoggerMixin): - DATE_FORMAT = "%Y-%m-%d" - YEAR_FORMAT = "%Y" - - unescape_entity_references = html.unescape - - @property - def xpath_expression(self) -> str: - return "//Item" - - parenthetical = re.compile(r" \([^)]+\)$") - - format_data_for_bibliotheca_format = frozendict( - { - "EPUB": (Representation.EPUB_MEDIA_TYPE, DeliveryMechanism.ADOBE_DRM), - "EPUB3": (Representation.EPUB_MEDIA_TYPE, DeliveryMechanism.ADOBE_DRM), - "PDF": (Representation.PDF_MEDIA_TYPE, DeliveryMechanism.ADOBE_DRM), - "MP3": (None, DeliveryMechanism.FINDAWAY_DRM), - } - ) - - @classmethod - def contributors_from_string( - cls, string: str | None, role: str = Contributor.Role.AUTHOR - ) -> list[ContributorData]: - contributors: list[ContributorData] = [] - if not string: - return contributors - - # Contributors may have two levels of entity reference escaping, - # one of which will have already been handled by the initial parse. - # We handle the potential need for a second unescaping here. - string = cls.unescape_entity_references(string) - - for sort_name in string.split(";"): - sort_name = cls.parenthetical.sub("", sort_name.strip()) - contributors.append( - ContributorData(sort_name=sort_name.strip(), roles=[role]) - ) - return contributors - - @classmethod - def parse_genre_string(self, s: str | None) -> list[SubjectData]: - genres: list[SubjectData] = [] - if not s: - return genres - for i in s.split(","): - i = i.strip() - if not i: - continue - i = ( - i.replace("&amp;", "&") - .replace("&", "&") - .replace("'", "'") - ) - genres.append( - SubjectData( - type=Subject.BISAC, - identifier=None, - name=i, - weight=Classification.TRUSTED_DISTRIBUTOR_WEIGHT, - ) - ) - return genres - - def process_one( - self, tag: _Element, namespaces: dict[str, str] | None - ) -> BibliographicData: - """Turn an tag into a BibliographicData and an encompassed CirculationData - objects, and return the BibliographicData.""" - - def value(bibliotheca_key: str) -> str | None: - return self.text_of_optional_subtag(tag, bibliotheca_key) - - primary_identifier = IdentifierData( - type=Identifier.BIBLIOTHECA_ID, identifier=value("ItemId") - ) - - identifiers = [] - for key in ("ISBN13", "PhysicalISBN"): - v = value(key) - if v: - identifiers.append(IdentifierData(type=Identifier.ISBN, identifier=v)) - - subjects = self.parse_genre_string(value("Genre")) - - title = value("Title") - subtitle = value("SubTitle") - publisher = value("Publisher") - language = value("Language") - - authors = list(self.contributors_from_string(value("Authors"))) - narrators = list( - self.contributors_from_string(value("Narrator"), Contributor.Role.NARRATOR) - ) - - published_date = None - published = value("PubDate") - if published: - formats = [self.DATE_FORMAT, self.YEAR_FORMAT] - else: - published = value("PubYear") - formats = [self.YEAR_FORMAT] - - for format in formats: - try: - published_date = strptime_utc(published, format) # type: ignore[arg-type] - except ValueError as e: - pass - - links = [] - description = value("Description") - if description: - links.append(LinkData(rel=Hyperlink.DESCRIPTION, content=description)) - - # Presume all images from Bibliotheca are JPEG. - media_type = Representation.JPEG_MEDIA_TYPE - cover_url = self.text_of_subtag(tag, "CoverLinkURL").replace("&", "&") - - # Unless the URL format has drastically changed, we should be - # able to generate a thumbnail URL based on the full-size - # cover URL found in the response document. - # - # NOTE: this is an undocumented feature of the Bibliotheca API - # which was discovered by investigating the BookLinkURL. - if "/delivery/img" in cover_url: - thumbnail_url = cover_url + "&size=NORMAL" - thumbnail = LinkData( - rel=Hyperlink.THUMBNAIL_IMAGE, href=thumbnail_url, media_type=media_type - ) - else: - thumbnail = None - cover_link = LinkData( - rel=Hyperlink.IMAGE, - href=cover_url, - media_type=media_type, - thumbnail=thumbnail, - ) - links.append(cover_link) - - alternate_url = self.text_of_subtag(tag, "BookLinkURL").replace("&", "&") - links.append(LinkData(rel="alternate", href=alternate_url)) - - measurements = [] - pages = value("NumberOfPages") - if pages: - pages_int = int(pages) - measurements.append( - MeasurementData( - quantity_measured=Measurement.PAGE_COUNT, value=pages_int - ) - ) - - circulation, medium = self._make_circulation_data(tag, primary_identifier) - - bibliographic = BibliographicData( - data_source_name=DataSource.BIBLIOTHECA, - title=title, - subtitle=subtitle, - language=language, - medium=medium, - publisher=publisher, - published=published_date, - primary_identifier_data=primary_identifier, - identifiers=identifiers, - subjects=subjects, - contributors=authors + narrators, - measurements=measurements, - links=links, - circulation=circulation, - ) - return bibliographic - - def _make_circulation_data( - self, tag: _Element, primary_identifier: IdentifierData - ) -> tuple[CirculationData, str]: - """Parse out a CirculationData containing current circulation - and formatting information. - """ - - def value(bibliotheca_key: str) -> str: - return self.text_of_subtag(tag, bibliotheca_key) - - def intvalue(key: str) -> int: - return self.int_of_subtag(tag, key) - - book_format = value("BookFormat") - medium, formats = self.internal_formats(book_format) - - licenses_owned = intvalue("TotalCopies") - try: - licenses_available = intvalue("AvailableCopies") - except IndexError: - self.log.warning( - "No information on available copies for %s", - primary_identifier.identifier, - ) - licenses_available = 0 - - patrons_in_hold_queue = intvalue("OnHoldCount") - licenses_reserved = 0 - - license_status = ( - LicensePoolStatus.ACTIVE - if licenses_owned > 0 - else LicensePoolStatus.EXHAUSTED - ) - - circulation = CirculationData( - data_source_name=DataSource.BIBLIOTHECA, - primary_identifier_data=primary_identifier, - licenses_owned=licenses_owned, - licenses_available=licenses_available, - licenses_reserved=licenses_reserved, - patrons_in_hold_queue=patrons_in_hold_queue, - formats=formats, - status=license_status, - ) - return circulation, medium - - @classmethod - def internal_formats(cls, book_format: str) -> tuple[str, list[FormatData]]: - """Convert the term Bibliotheca uses to refer to a book - format into a (medium [formats]) 2-tuple. - """ - if book_format not in cls.format_data_for_bibliotheca_format: - cls.logger().error("Unrecognized BookFormat: %s", book_format) - return Edition.BOOK_MEDIUM, [] - - content_type, drm_scheme = cls.format_data_for_bibliotheca_format[book_format] - - format = FormatData(content_type=content_type, drm_scheme=drm_scheme) - if book_format == "MP3": - medium = Edition.AUDIO_MEDIUM - else: - medium = Edition.BOOK_MEDIUM - return medium, [format] - - -class BibliothecaParser[T](XMLProcessor[T], ABC): - INPUT_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S" - - @classmethod - def parse_date(cls, value: str | None) -> datetime | None: - """Parse the string Bibliotheca sends as a date. - - Usually this is a string in INPUT_TIME_FORMAT, but it might be None. - """ - if not value: - parsed = None - else: - try: - parsed = strptime_utc(value, cls.INPUT_TIME_FORMAT) - except ValueError as e: - logging.error( - 'Unable to parse Bibliotheca date: "%s"', value, exc_info=e - ) - parsed = None - return to_utc(parsed) - - @overload - def date_from_subtag(self, tag: _Element, key: str) -> datetime: ... - - @overload - def date_from_subtag( - self, tag: _Element, key: str, required: Literal[False] - ) -> datetime | None: ... - - def date_from_subtag( - self, tag: _Element, key: str, required: bool = True - ) -> datetime | None: - value = ( - self.text_of_subtag(tag, key) - if required - else self.text_of_optional_subtag(tag, key) - ) - return self.parse_date(value) - - -class ErrorParser(BibliothecaParser[BaseProblemDetailException]): - """Turn an error document from the Bibliotheca web service into a CheckoutException""" - - wrong_status = re.compile( - "the patron document status was ([^ ]+) and not one of ([^ ]+)" - ) - - loan_limit_reached = re.compile("Patron cannot loan more than [0-9]+ document") - - hold_limit_reached = re.compile("Patron cannot have more than [0-9]+ hold") - - error_mapping = { - "The patron does not have the book on hold": NotOnHold, - "The patron has no eBooks checked out": NotCheckedOut, - } - - @property - def xpath_expression(self) -> str: - return "//Error" - - def process_first(self, string: str | bytes) -> BaseProblemDetailException: - try: - return_val = super().process_first(string) - except Exception as e: - # The server sent us an error with an incorrect or - # nonstandard syntax. - if isinstance(string, bytes): - try: - debug = string.decode("utf-8") - except UnicodeDecodeError: - debug = "Unreadable error message (Unicode decode error)." - else: - debug = string - return RemoteInitiatedServerError(debug, BibliothecaAPI.SERVICE_NAME) - - if return_val is None: - # We were not able to interpret the result as an error. - # The most likely cause is that the Bibliotheca app server is down. - return RemoteInitiatedServerError( - "Unknown error", - BibliothecaAPI.SERVICE_NAME, - ) - - return return_val - - def process_one( - self, error_tag: _Element, namespaces: dict[str, str] | None - ) -> BaseProblemDetailException: - message = self.text_of_optional_subtag(error_tag, "Message") - if not message: - return RemoteInitiatedServerError( - "Unknown error", - BibliothecaAPI.SERVICE_NAME, - ) - - if message in self.error_mapping: - return self.error_mapping[message](message) - if message in ("Authentication failed", "Unknown error"): - # 'Unknown error' is an unknown error on the Bibliotheca side. - # - # 'Authentication failed' could _in theory_ be an error on - # our side, but if authentication is set up improperly we - # actually get a 401 and no body. When we get a real error - # document with 'Authentication failed', it's always a - # transient error on the Bibliotheca side. Possibly some - # authentication internal to Bibliotheca has failed? Anyway, it - # happens relatively frequently. - return RemoteInitiatedServerError(message, BibliothecaAPI.SERVICE_NAME) - - m = self.loan_limit_reached.search(message) - if m: - return PatronLoanLimitReached(message) - - m = self.hold_limit_reached.search(message) - if m: - return PatronHoldLimitReached(message) - - m = self.wrong_status.search(message) - if not m: - return RemoteInitiatedServerError(message, BibliothecaAPI.SERVICE_NAME) - actual, expected = m.groups() - expected = expected.split(",") - - if actual == "CAN_WISH": - return NoLicenses(debug_info=message) - - if "CAN_LOAN" in expected and actual == "CAN_HOLD": - return NoAvailableCopies(debug_info=message) - - if "CAN_LOAN" in expected and actual == "HOLD": - return AlreadyOnHold(debug_info=message) - - if "CAN_LOAN" in expected and actual == "LOAN": - return AlreadyCheckedOut(debug_info=message) - - if "CAN_HOLD" in expected and actual == "CAN_LOAN": - return CurrentlyAvailable(debug_info=message) - - if "CAN_HOLD" in expected and actual == "HOLD": - return AlreadyOnHold(debug_info=message) - - if "CAN_HOLD" in expected: - return CannotHold(debug_info=message) - - if "CAN_LOAN" in expected: - return CannotLoan(debug_info=message) - - return RemoteInitiatedServerError(message, BibliothecaAPI.SERVICE_NAME) - - -class PatronCirculationParser(XMLParser): - """Parse Bibliotheca's patron circulation status document into a list of - LoanInfo and HoldInfo objects. - """ - - id_type = Identifier.BIBLIOTHECA_ID - - def __init__(self, collection: Collection) -> None: - self.collection = collection - - def process_all(self, string: bytes | str) -> itertools.chain[LoanInfo | HoldInfo]: - xml = self._load_xml(string) - loans = self._process_all( - xml, "//Checkouts/Item", namespaces={}, handler=self.process_one_loan - ) - holds = self._process_all( - xml, "//Holds/Item", namespaces={}, handler=self.process_one_hold - ) - reserves = self._process_all( - xml, "//Reserves/Item", namespaces={}, handler=self.process_one_reserve - ) - return itertools.chain(loans, holds, reserves) - - def process_one_loan( - self, tag: _Element, namespaces: dict[str, str] - ) -> LoanInfo | None: - return self.process_one(tag, namespaces, LoanInfo) - - def process_one_hold( - self, tag: _Element, namespaces: dict[str, str] - ) -> HoldInfo | None: - return self.process_one(tag, namespaces, HoldInfo) - - def process_one_reserve( - self, tag: _Element, namespaces: dict[str, str] - ) -> HoldInfo | None: - hold_info = self.process_one(tag, namespaces, HoldInfo) - if hold_info is not None: - hold_info.hold_position = 0 - return hold_info - - def process_one[T]( - self, tag: _Element, namespaces: dict[str, str], source_class: type[T] - ) -> T | None: - if not tag.xpath("ItemId"): - # This happens for events associated with books - # no longer in our collection. - return None - - def datevalue(key: str) -> datetime: - value = self.text_of_subtag(tag, key) - return strptime_utc(value, BibliothecaAPI.ARGUMENT_TIME_FORMAT) - - identifier = self.text_of_subtag(tag, "ItemId") - start_date = datevalue("EventStartDateInUTC") - end_date = datevalue("EventEndDateInUTC") - kwargs = { - "collection_id": self.collection.id, - "identifier_type": self.id_type, - "identifier": identifier, - "start_date": start_date, - "end_date": end_date, - } - if source_class is HoldInfo: - kwargs["hold_position"] = self.int_of_subtag(tag, "Position") - return source_class(**kwargs) - - -class DateResponseParser(BibliothecaParser[Optional[datetime]], ABC): - """Extract a date from a response.""" - - RESULT_TAG_NAME: str | None = None - DATE_TAG_NAME: str | None = None - - @property - def xpath_expression(self) -> str: - return f"/{self.RESULT_TAG_NAME}/{self.DATE_TAG_NAME}" - - def process_one( - self, tag: _Element, namespaces: dict[str, str] | None - ) -> datetime | None: - due_date = tag.text - if not due_date: - return None - return strptime_utc(due_date, EventParser.INPUT_TIME_FORMAT) - - -class CheckoutResponseParser(DateResponseParser): - """Extract due date from a checkout response.""" - - @property - def xpath_expression(self) -> str: - return f"/CheckoutResult/DueDateInUTC" - - -class HoldResponseParser(DateResponseParser): - """Extract availability date from a hold response.""" - - @property - def xpath_expression(self) -> str: - return f"/PlaceHoldResult/AvailabilityDateInUTC" - - -class EventParser( - BibliothecaParser[tuple[str, str, str | None, datetime, datetime | None, str]] -): - """Parse Bibliotheca's event file format into our native event objects.""" - - EVENT_SOURCE = "Bibliotheca" - - SET_DELIVERY_MECHANISM_AT = BaseCirculationAPI.BORROW_STEP - - # Map Bibliotheca's event names to our names. - EVENT_NAMES = { - "CHECKOUT": CirculationEvent.DISTRIBUTOR_CHECKOUT, - "CHECKIN": CirculationEvent.DISTRIBUTOR_CHECKIN, - "HOLD": CirculationEvent.DISTRIBUTOR_HOLD_PLACE, - "RESERVED": CirculationEvent.DISTRIBUTOR_AVAILABILITY_NOTIFY, - "PURCHASE": CirculationEvent.DISTRIBUTOR_LICENSE_ADD, - "REMOVED": CirculationEvent.DISTRIBUTOR_LICENSE_REMOVE, - } - - @property - def xpath_expression(self) -> str: - return "//CloudLibraryEvent" - - def process_all( - self, string: bytes | str, no_events_error: bool = False - ) -> Generator[tuple[str, str, str | None, datetime, datetime | None, str]]: - has_events = False - # Bibliotheca occasionally returns an empty response body. An empty - # document cannot be parsed as XML (lxml raises "Document is empty" - # even in recovery mode), so treat a blank body the same as a - # response that contained no events rather than letting the parse - # error propagate as an unhandled exception. - if string.strip(): - for i in super().process_all(string): - yield i - has_events = True - - # If we are catching up on events and we expect to have a time - # period where there are no events, we don't want to consider that - # action as an error. By default, not having events is not - # considered to be an error. - if not has_events and no_events_error: - # An empty list of events may mean nothing happened, or it - # may indicate an unreported server-side error. To be - # safe, we'll treat this as a server-initiated error - # condition. If this is just a slow day, normal behavior - # will resume as soon as something happens. - raise RemoteInitiatedServerError( - "No events returned from server. This may not be an error, but treating it as one to be safe.", - BibliothecaAPI.SERVICE_NAME, - ) - - def process_one( - self, tag: _Element, namespaces: dict[str, str] | None - ) -> tuple[str, str, str | None, datetime, datetime | None, str]: - isbn = self.text_of_subtag(tag, "ISBN") - bibliotheca_id = self.text_of_subtag(tag, "ItemId") - patron_id = self.text_of_optional_subtag(tag, "PatronId") - - start_time = self.date_from_subtag(tag, "EventStartDateTimeInUTC") - end_time = self.date_from_subtag(tag, "EventEndDateTimeInUTC", required=False) - - bibliotheca_event_type = self.text_of_subtag(tag, "EventType") - internal_event_type = self.EVENT_NAMES[bibliotheca_event_type] - - return ( - bibliotheca_id, - isbn, - patron_id, - start_time, - end_time, - internal_event_type, - ) - - -BibliothecaApiClassT = type[BibliothecaAPI] | BibliothecaAPI - - -class BibliothecaBibliographicCoverageProvider(BibliographicCoverageProvider): - """Fill in bibliographic metadata for Bibliotheca records. - - This will occasionally fill in some availability information for a - single Collection, but we rely on Monitors to keep availability - information up to date for all Collections. - """ - - SERVICE_NAME = "Bibliotheca Bibliographic Coverage Provider" - DATA_SOURCE_NAME = DataSource.BIBLIOTHECA - PROTOCOL = BibliothecaAPI.label() - INPUT_IDENTIFIER_TYPES = Identifier.BIBLIOTHECA_ID - - # 25 is the maximum batch size for the Bibliotheca API. - DEFAULT_BATCH_SIZE = 25 - - def __init__( - self, - collection: Collection, - api_class: BibliothecaApiClassT = BibliothecaAPI, - **kwargs: Any, - ) -> None: - """Constructor. - - :param collection: Provide bibliographic coverage to all - Bibliotheca books in the given Collection. - :param api_class: Instantiate this class with the given Collection, - rather than instantiating BibliothecaAPI. - :param input_identifiers: Passed in by RunCoverageProviderScript. - A list of specific identifiers to get coverage for. - """ - super().__init__(collection, **kwargs) - if isinstance(api_class, BibliothecaAPI): - # This is an already instantiated API object. Use it - # instead of creating a new one. - self.api = api_class - else: - # A web application should not use this option because it - # will put a non-scoped session in the mix. - _db = Session.object_session(collection) - self.api = api_class(_db, collection) - - def process_item(self, identifier: Identifier) -> Identifier | CoverageFailure: - list_bibliographic = self.api.bibliographic_lookup(identifier) - if not list_bibliographic: - return self.failure(identifier, "Bibliotheca bibliographic lookup failed.") # type: ignore[no-any-return] - [bibliographic] = list_bibliographic - return self.set_bibliographic(identifier, bibliographic) # type: ignore[no-any-return] diff --git a/src/palace/manager/integration/license/bibliotheca/__init__.py b/src/palace/manager/integration/license/bibliotheca/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/palace/manager/integration/license/bibliotheca/api.py b/src/palace/manager/integration/license/bibliotheca/api.py new file mode 100644 index 0000000000..dc09b91b78 --- /dev/null +++ b/src/palace/manager/integration/license/bibliotheca/api.py @@ -0,0 +1,604 @@ +from __future__ import annotations + +import hashlib +import hmac +import json +import time +import urllib.parse +from collections.abc import Collection as CollectionT, Generator, Iterable +from datetime import datetime, timedelta +from io import BytesIO +from typing import Any, Unpack + +from lxml.etree import Error +from pymarc import Record, parse_xml_to_array +from requests import Response +from sqlalchemy.orm import Session + +from palace.util.datetime_helpers import utc_now + +from palace.manager.api.circulation.base import ( + BaseCirculationAPI, + PatronActivityCirculationAPI, +) +from palace.manager.api.circulation.data import HoldInfo, LoanInfo +from palace.manager.api.circulation.exceptions import ( + AlreadyCheckedOut, + CannotHold, + CannotReleaseHold, + RemoteInitiatedServerError, +) +from palace.manager.api.circulation.fulfillment import DirectFulfillment +from palace.manager.api.selftest import HasCollectionSelfTests +from palace.manager.api.web_publication_manifest import FindawayManifest, SpineItem +from palace.manager.core.config import CannotLoadConfiguration +from palace.manager.core.selftest import SelfTestResult +from palace.manager.data_layer.bibliographic import BibliographicData +from palace.manager.integration.license.bibliotheca.constants import ( + BIBLIOTHECA_LABEL, + BIBLIOTHECA_SERVICE_NAME, + BIBLIOTHECA_TIME_FORMAT, +) +from palace.manager.integration.license.bibliotheca.parser import ( + CheckoutResponseParser, + ErrorParser, + EventParser, + HoldResponseParser, + ItemListParser, + PatronCirculationParser, +) +from palace.manager.integration.license.bibliotheca.settings import ( + BibliothecaLibrarySettings, + BibliothecaSettings, +) +from palace.manager.sqlalchemy.model.collection import Collection +from palace.manager.sqlalchemy.model.datasource import DataSource +from palace.manager.sqlalchemy.model.identifier import Identifier +from palace.manager.sqlalchemy.model.licensing import ( + DeliveryMechanism, + LicensePool, + LicensePoolDeliveryMechanism, +) +from palace.manager.sqlalchemy.model.patron import Patron +from palace.manager.sqlalchemy.model.resource import Representation +from palace.manager.util import base64 +from palace.manager.util.http.exception import RemoteIntegrationException +from palace.manager.util.http.http import HTTP, RequestKwargs + + +class BibliothecaAPI( + PatronActivityCirculationAPI[BibliothecaSettings, BibliothecaLibrarySettings], + HasCollectionSelfTests, +): + AUTH_TIME_FORMAT = "%a, %d %b %Y %H:%M:%S GMT" + ARGUMENT_TIME_FORMAT = BIBLIOTHECA_TIME_FORMAT + AUTHORIZATION_FORMAT = "3MCLAUTH %s:%s" + + DATETIME_HEADER = "3mcl-Datetime" + AUTHORIZATION_HEADER = "3mcl-Authorization" + VERSION_HEADER = "3mcl-Version" + + DEFAULT_VERSION = "2.0" + DEFAULT_BASE_URL = "https://partner.yourcloudlibrary.com/" + + MAX_AGE = timedelta(days=730).seconds + CAN_REVOKE_HOLD_WHEN_RESERVED = False + SET_DELIVERY_MECHANISM_AT = None + + SERVICE_NAME = BIBLIOTHECA_SERVICE_NAME + + @classmethod + def settings_class(cls) -> type[BibliothecaSettings]: + return BibliothecaSettings + + @classmethod + def library_settings_class(cls) -> type[BibliothecaLibrarySettings]: + return BibliothecaLibrarySettings + + @classmethod + def label(cls) -> str: + return BIBLIOTHECA_LABEL + + @classmethod + def description(cls) -> str: + return "" + + def __init__(self, _db: Session, collection: Collection) -> None: + super().__init__(_db, collection) + + self._db = _db + self.version = self.DEFAULT_VERSION + self.account_id = self.settings.username + self.account_key = self.settings.password + self.library_id = self.settings.external_account_id + self.base_url = self.DEFAULT_BASE_URL + + if not self.account_id or not self.account_key or not self.library_id: + raise CannotLoadConfiguration("Bibliotheca configuration is incomplete.") + + self.item_list_parser = ItemListParser() + self.collection_id = collection.id + + @property + def data_source(self) -> DataSource: + return DataSource.lookup(self._db, DataSource.BIBLIOTHECA, autocreate=True) + + def now(self) -> str: + """Return the current GMT time in the format 3M expects.""" + return time.strftime(self.AUTH_TIME_FORMAT, time.gmtime()) + + def sign(self, method: str, headers: dict[str, str], path: str) -> None: + """Add appropriate headers to a request.""" + authorization, now = self.authorization(method, path) + headers[self.DATETIME_HEADER] = now + headers[self.VERSION_HEADER] = self.version + headers[self.AUTHORIZATION_HEADER] = authorization + + def authorization(self, method: str, path: str) -> tuple[str, str]: + signature, now = self.signature(method, path) + auth = self.AUTHORIZATION_FORMAT % (self.account_id, signature) + return auth, now + + def signature(self, method: str, path: str) -> tuple[str, str]: + now = self.now() + signature_components = [now, method, path] + signature_string = "\n".join(signature_components) + digest = hmac.new( + self.account_key.encode("utf-8"), + msg=signature_string.encode("utf-8"), + digestmod=hashlib.sha256, + ).digest() + signature = base64.standard_b64encode(digest) + return signature, now + + def full_url(self, path: str) -> str: + if not path.startswith("/cirrus"): + path = self.full_path(path) + return urllib.parse.urljoin(self.base_url, path) + + def full_path(self, path: str) -> str: + if not path.startswith("/"): + path = "/" + path + if not path.startswith("/cirrus"): + path = f"/cirrus/library/{self.library_id}{path}" + return path + + def request( + self, + path: str, + body: str | None = None, + method: str = "GET", + ) -> Response: + path = self.full_path(path) + url = self.full_url(path) + if method == "GET": + headers = {"Accept": "application/xml"} + else: + headers = {"Content-Type": "application/xml"} + self.sign(method, headers, path) + return self._request_with_timeout( + method, + url, + data=body, + headers=headers, + allow_redirects=False, + timeout=60, + ) + + def marc_request( + self, start: datetime, end: datetime, offset: int = 1, limit: int = 50 + ) -> Generator[Record]: + """Make an HTTP request to look up the MARC records for books purchased + between two given dates. + + :param start: A datetime to start looking for purchases. + :param end: A datetime to stop looking for purchases. + :param offset: An offset used to paginate results. + :param limit: A limit used to paginate results. + :raise: An appropriate exception if the request returned a non-200 + status code. An empty response body is not an error: it is treated + as "no records" and the generator simply yields nothing. + :yield: A list of MARC records. + """ + start_param = start.strftime(self.ARGUMENT_TIME_FORMAT) + end_param = end.strftime(self.ARGUMENT_TIME_FORMAT) + url = "data/marc?startdate=%s&enddate=%s&offset=%d&limit=%d" % ( + start_param, + end_param, + offset, + limit, + ) + response = self.request(url) + if response.status_code != 200: + raise ErrorParser().process_first(response.content) + if not response.content.strip(): + # Bibliotheca sometimes returns an empty body (HTTP 200 with no + # XML document) for a window that contains no purchase records. + # pymarc's parse_xml_to_array raises SAXException("no element + # found") on an empty document, so treat an empty body as "no + # records" and yield nothing rather than letting it propagate. + self.log.info( + f"Bibliotheca MARC request to '{url}' returned an empty " + "response body; treating as no records." + ) + return + yield from parse_xml_to_array(BytesIO(response.content)) + + def bibliographic_lookup_request(self, identifiers: CollectionT[str]) -> bytes: + """Make an HTTP request to look up current bibliographic and + circulation information for the given `identifiers`. + + :param identifiers: Strings containing Bibliotheca identifiers. + :return: A string containing an XML document, or None if there was + an error not handled as an exception. + """ + url = "/items/" + ",".join(identifiers) + response = self.request(url) + return response.content + + def bibliographic_lookup( + self, identifiers: CollectionT[str | Identifier] | str | Identifier + ) -> list[BibliographicData]: + """Look up current bibliographic and circulation information for the + given `identifiers`. + + :param identifiers: A list containing either Identifier + objects or Bibliotheca identifier strings. + :raise RemoteInitiatedServerError: If Bibliotheca returns an empty + response body (HTTP 200 with no XML document). See the comment + below for why this is treated as a transient error rather than as + "no items found". + """ + identifiers_list = ( + [identifiers] + if isinstance(identifiers, Identifier) or isinstance(identifiers, str) + else identifiers + ) + identifier_strings = [] + for i in identifiers_list: + if isinstance(i, Identifier): + i = i.identifier + identifier_strings.append(i) + + data = self.bibliographic_lookup_request(identifier_strings) + if not data.strip(): + # Bibliotheca occasionally returns an empty body (HTTP 200 with no + # XML document). An empty document cannot be parsed as XML (lxml + # raises "Document is empty" even in recovery mode). Unlike a + # well-formed document that simply omits some of the requested + # items, an empty body tells us nothing about which titles still + # exist, so we must not treat it as "no items returned": that would + # make BibliothecaCirculationUpdater._process_batch zero out the + # availability of every requested identifier as if it had been + # removed from circulation. Instead, raise a transient remote error + # -- mirroring how marc_request/ErrorParser treat empty or malformed + # Bibliotheca responses -- so the caller retries rather than + # corrupting availability data. + raise RemoteInitiatedServerError( + "Bibliotheca returned an empty response body for a bibliographic lookup.", + self.SERVICE_NAME, + ) + return [ + bibliographic for bibliographic in self.item_list_parser.process_all(data) + ] + + def _request_with_timeout( + self, http_method: str, url: str, **kwargs: Unpack[RequestKwargs] + ) -> Response: + """This will be overridden in MockBibliothecaAPI.""" + return HTTP.request_with_timeout(http_method, url, **kwargs) + + def _run_self_tests(self, _db: Session) -> Generator[SelfTestResult]: + def _count_events() -> str: + now = utc_now() + five_minutes_ago = now - timedelta(minutes=5) + count = len(list(self.get_events_between(five_minutes_ago, now))) + return "Found %d event(s)" % count + + yield self.run_test( + "Asking for circulation events for the last five minutes", _count_events + ) + + for result in self.default_patrons(self.collection): + if isinstance(result, SelfTestResult): + yield result + continue + library, patron, pin = result + + def _count_activity() -> str: + result = self.patron_activity(patron, pin) + return "Found %d loans/holds" % len(list(result)) + + yield self.run_test( + "Checking activity for test patron for library %s" % library.name, + _count_activity, + ) + + def get_events_between( + self, start: datetime, end: datetime, no_events_error: bool = False + ) -> Generator[tuple[str, str, str | None, datetime, datetime | None, str]]: + """Return event objects for events between the given times.""" + start_str = start.strftime(self.ARGUMENT_TIME_FORMAT) + end_str = end.strftime(self.ARGUMENT_TIME_FORMAT) + url = f"data/cloudevents?startdate={start_str}&enddate={end_str}" + response = self.request(url) + try: + events = EventParser().process_all(response.content, no_events_error) + except Exception as e: + self.log.error( + "Error parsing Bibliotheca response content: %s", + response.content, + exc_info=e, + ) + raise e + return events + + def update_availability(self, licensepool: LicensePool) -> None: + """Update the availability information for a single LicensePool.""" + # Local import to avoid circular dependency between api.py and + # circulation_updater.py (the updater imports BibliothecaAPI). + from palace.manager.integration.license.bibliotheca.circulation_updater import ( + BibliothecaCirculationUpdater, + ) + + updater = BibliothecaCirculationUpdater( + self._db, licensepool.collection, api=self + ) + updater.process_identifiers([licensepool.identifier]) + + def _patron_activity_request(self, patron: Patron) -> Response: + patron_id = patron.authorization_identifier + path = "circulation/patron/%s" % patron_id + return self.request(path) + + def patron_activity( + self, patron: Patron, pin: str | None + ) -> Iterable[LoanInfo | HoldInfo]: + response = self._patron_activity_request(patron) + try: + return PatronCirculationParser(self.collection).process_all( + response.content + ) + except Error as e: + # XML parse error from remote. + raise RemoteIntegrationException( + response.url, "Unable to parse response XML." + ) from e + + TEMPLATE = "<%(request_type)s>%(item_id)s%(patron_id)s" + + def checkout( + self, + patron_obj: Patron, + patron_password: str | None, + licensepool: LicensePool, + delivery_mechanism: LicensePoolDeliveryMechanism | None, + ) -> LoanInfo: + """Check out a book on behalf of a patron. + + :param patron_obj: a Patron object for the patron who wants + to check out the book. + + :param patron_password: The patron's alleged password. Not used here + since Bibliotheca trusts Simplified to do the check ahead of time. + + :param licensepool: LicensePool for the book to be checked out. + + :return: a LoanInfo object + """ + bibliotheca_id = licensepool.identifier.identifier + patron_identifier = patron_obj.authorization_identifier + args = dict( + request_type="CheckoutRequest", + item_id=bibliotheca_id, + patron_id=patron_identifier, + ) + body = self.TEMPLATE % args + response = self.request("checkout", body, method="PUT") + if response.status_code == 201: + # New loan + start_date = utc_now() + elif response.status_code == 200: + # Old loan -- we don't know the start date + start_date = None + else: + # Error condition. + error = ErrorParser().process_first(response.content) + if isinstance(error, AlreadyCheckedOut): + # It's already checked out. No problem. + pass + else: + raise error + + # At this point we know we have a loan. + loan_expires = CheckoutResponseParser().process_first(response.content) + loan = LoanInfo.from_license_pool( + licensepool, + end_date=loan_expires, + ) + return loan + + def fulfill( + self, + patron: Patron, + password: str, + pool: LicensePool, + delivery_mechanism: LicensePoolDeliveryMechanism, + **kwargs: Unpack[BaseCirculationAPI.FulfillKwargs], + ) -> DirectFulfillment: + """Get the actual resource file to the patron.""" + if ( + delivery_mechanism.delivery_mechanism.drm_scheme + == DeliveryMechanism.FINDAWAY_DRM + ): + fulfill_method = self.get_audio_fulfillment_file + content_transformation = self.findaway_license_to_webpub_manifest + else: + fulfill_method = self.get_fulfillment_file + content_transformation = None + response = fulfill_method( + patron.authorization_identifier, pool.identifier.identifier + ) + content: str | bytes = response.content + content_type = None + if content_transformation: + try: + content_type, content = content_transformation(pool, content) + except Exception as e: + self.log.error( + "Error transforming fulfillment document: %s", + response.content, + exc_info=e, + ) + return DirectFulfillment( + content=content, + content_type=content_type or response.headers.get("Content-Type"), + ) + + def get_fulfillment_file( + self, patron_id: str | None, bibliotheca_id: str + ) -> Response: + args = dict( + request_type="ACSMRequest", item_id=bibliotheca_id, patron_id=patron_id + ) + body = self.TEMPLATE % args + return self.request("GetItemACSM", body, method="PUT") + + def get_audio_fulfillment_file( + self, patron_id: str | None, bibliotheca_id: str + ) -> Response: + args = dict( + request_type="AudioFulfillmentRequest", + item_id=bibliotheca_id, + patron_id=patron_id, + ) + body = self.TEMPLATE % args + return self.request("GetItemAudioFulfillment", body, method="POST") + + def checkin(self, patron: Patron, pin: str, licensepool: LicensePool) -> None: + patron_id = patron.authorization_identifier + item_id = licensepool.identifier.identifier + args = dict(request_type="CheckinRequest", item_id=item_id, patron_id=patron_id) + body = self.TEMPLATE % args + self.request("checkin", body, method="PUT") + + def place_hold( + self, + patron: Patron, + pin: str | None, + licensepool: LicensePool, + notification_email_address: str | None = None, + ) -> HoldInfo: + """Place a hold. + + :return: a HoldInfo object. + """ + patron_id = patron.authorization_identifier + item_id = licensepool.identifier.identifier + args = dict( + request_type="PlaceHoldRequest", item_id=item_id, patron_id=patron_id + ) + body = self.TEMPLATE % args + response = self.request("placehold", body, method="PUT") + # The response comes in as a byte string that we must + # convert into a string. + response_content = None + if response.content: + response_content = response.content.decode("utf-8") + if response.status_code in (200, 201): + start_date = utc_now() + end_date = HoldResponseParser().process_first(response_content) + return HoldInfo.from_license_pool( + licensepool, + start_date=start_date, + end_date=end_date, + hold_position=None, + ) + else: + if not response_content: + raise CannotHold() + error = ErrorParser().process_first(response_content) + if isinstance(error, Exception): + raise error + else: + raise CannotHold(error) + + def release_hold(self, patron: Patron, pin: str, licensepool: LicensePool) -> None: + patron_id = patron.authorization_identifier + item_id = licensepool.identifier.identifier + args = dict( + request_type="CancelHoldRequest", item_id=item_id, patron_id=patron_id + ) + body = self.TEMPLATE % args + response = self.request("cancelhold", body, method="PUT") + if response.status_code not in (200, 404): + raise CannotReleaseHold() + + @classmethod + def findaway_license_to_webpub_manifest( + cls, license_pool: LicensePool, findaway_license: str | bytes | dict[str, Any] + ) -> tuple[str, str]: + """Convert a Bibliotheca license document to a FindawayManifest + suitable for serving to a mobile client. + + :param license_pool: A LicensePool for the title in question. + This will be used to fill in basic bibliographic information. + + :param findaway_license: A string containing a Findaway + license document via Bibliotheca, or a dictionary + representing such a document loaded into JSON form. + """ + if isinstance(findaway_license, (bytes, str)): + findaway_license = json.loads(findaway_license) + assert isinstance( + findaway_license, dict + ), "Expected a JSON object, got %s" % type(findaway_license) + + kwargs = {} + for findaway_extension in [ + "accountId", + "checkoutId", + "fulfillmentId", + "licenseId", + "sessionKey", + ]: + value = findaway_license.get(findaway_extension, None) + kwargs[findaway_extension] = value + + # Create the SpineItem objects. + audio_format = findaway_license.get("format") + if audio_format == "MP3": + part_media_type = Representation.MP3_MEDIA_TYPE + else: + cls.logger().error( + "Unknown Findaway audio format encountered: %s", audio_format + ) + part_media_type = None + + spine_items = [] + for part in findaway_license["items"]: + title = part.get("title") + + # TODO: Incoming duration appears to be measured in + # milliseconds. This assumption makes our example + # audiobook take about 7.9 hours, and no other reasonable + # assumption is in the right order of magnitude. But this + # needs to be explicitly verified. + duration = part.get("duration", 0) / 1000.0 + + part_number = int(part.get("part", 0)) + + sequence = int(part.get("sequence", 0)) + + spine_items.append(SpineItem(title, duration, part_number, sequence)) + + # Create a FindawayManifest object and then convert it + # to a string. + manifest = FindawayManifest( + license_pool=license_pool, spine_items=spine_items, **kwargs + ) + + return DeliveryMechanism.FINDAWAY_DRM, str(manifest) + + +BibliothecaApiClassT = type[BibliothecaAPI] | BibliothecaAPI diff --git a/src/palace/manager/integration/license/bibliotheca_circulation_updater.py b/src/palace/manager/integration/license/bibliotheca/circulation_updater.py similarity index 98% rename from src/palace/manager/integration/license/bibliotheca_circulation_updater.py rename to src/palace/manager/integration/license/bibliotheca/circulation_updater.py index f37ce5d39a..4b6feba5dd 100644 --- a/src/palace/manager/integration/license/bibliotheca_circulation_updater.py +++ b/src/palace/manager/integration/license/bibliotheca/circulation_updater.py @@ -13,7 +13,7 @@ from palace.manager.celery.tasks import apply from palace.manager.data_layer.policy.replacement import ReplacementPolicy -from palace.manager.integration.license.bibliotheca import BibliothecaAPI +from palace.manager.integration.license.bibliotheca.api import BibliothecaAPI from palace.manager.sqlalchemy.constants import DataSourceConstants from palace.manager.sqlalchemy.model.collection import Collection from palace.manager.sqlalchemy.model.coverage import Timestamp @@ -157,7 +157,7 @@ def update_batch(self, offset: int = 0) -> BatchUpdateResult: def process_identifiers(self, identifiers: Sequence[Identifier]) -> None: """Process a caller-supplied list of identifiers without touching the Timestamp. - Used by :meth:`~palace.manager.integration.license.bibliotheca.BibliothecaAPI.update_availability` + Used by :meth:`~palace.manager.integration.license.bibliotheca.api.BibliothecaAPI.update_availability` and :class:`~palace.manager.scripts.availability.AvailabilityRefreshScript` for on-demand single-title refreshes. Reconciles the same way as :meth:`update_batch` — bibliographic metadata is hash-deduplicated and @@ -168,7 +168,7 @@ def process_identifiers(self, identifiers: Sequence[Identifier]) -> None: in the caller's session rather than queued as ``bibliographic_apply`` / ``circulation_apply`` tasks, so the updated availability is visible as soon as this method returns. Callers such - as :meth:`~palace.manager.integration.license.bibliotheca.BibliothecaAPI.update_availability` + as :meth:`~palace.manager.integration.license.bibliotheca.api.BibliothecaAPI.update_availability` rely on this — e.g. the circulation dispatcher reads ``LicensePool.licenses_available`` immediately after requesting an availability refresh. diff --git a/src/palace/manager/integration/license/bibliotheca/constants.py b/src/palace/manager/integration/license/bibliotheca/constants.py new file mode 100644 index 0000000000..4833cbe29b --- /dev/null +++ b/src/palace/manager/integration/license/bibliotheca/constants.py @@ -0,0 +1,13 @@ +from __future__ import annotations + +from palace.manager.sqlalchemy.constants import DataSourceConstants + +BIBLIOTHECA_LABEL = DataSourceConstants.BIBLIOTHECA + +# The name we use to identify Bibliotheca as the remote service, in self-test +# output and in the RemoteInitiatedServerErrors we raise against it. +BIBLIOTHECA_SERVICE_NAME = "Bibliotheca" + +# The format Bibliotheca uses for datetimes, both for the arguments it expects +# on requests and for the timestamps in the documents it returns. +BIBLIOTHECA_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S" diff --git a/src/palace/manager/integration/license/bibliotheca/coverage.py b/src/palace/manager/integration/license/bibliotheca/coverage.py new file mode 100644 index 0000000000..dec49fba26 --- /dev/null +++ b/src/palace/manager/integration/license/bibliotheca/coverage.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +from typing import Any + +from sqlalchemy.orm import Session + +from palace.manager.core.coverage import BibliographicCoverageProvider, CoverageFailure +from palace.manager.integration.license.bibliotheca.api import ( + BibliothecaAPI, + BibliothecaApiClassT, +) +from palace.manager.integration.license.bibliotheca.constants import BIBLIOTHECA_LABEL +from palace.manager.sqlalchemy.model.collection import Collection +from palace.manager.sqlalchemy.model.datasource import DataSource +from palace.manager.sqlalchemy.model.identifier import Identifier + + +class BibliothecaBibliographicCoverageProvider(BibliographicCoverageProvider): + """Fill in bibliographic metadata for Bibliotheca records. + + This will occasionally fill in some availability information for a + single Collection, but we rely on Monitors to keep availability + information up to date for all Collections. + """ + + SERVICE_NAME = "Bibliotheca Bibliographic Coverage Provider" + DATA_SOURCE_NAME = DataSource.BIBLIOTHECA + PROTOCOL = BIBLIOTHECA_LABEL + INPUT_IDENTIFIER_TYPES = Identifier.BIBLIOTHECA_ID + + # 25 is the maximum batch size for the Bibliotheca API. + DEFAULT_BATCH_SIZE = 25 + + def __init__( + self, + collection: Collection, + api_class: BibliothecaApiClassT = BibliothecaAPI, + **kwargs: Any, + ) -> None: + """Constructor. + + :param collection: Provide bibliographic coverage to all + Bibliotheca books in the given Collection. + :param api_class: Instantiate this class with the given Collection, + rather than instantiating BibliothecaAPI. + :param input_identifiers: Passed in by RunCoverageProviderScript. + A list of specific identifiers to get coverage for. + """ + super().__init__(collection, **kwargs) + if isinstance(api_class, BibliothecaAPI): + # This is an already instantiated API object. Use it + # instead of creating a new one. + self.api = api_class + else: + # A web application should not use this option because it + # will put a non-scoped session in the mix. + _db = Session.object_session(collection) + self.api = api_class(_db, collection) + + def process_item(self, identifier: Identifier) -> Identifier | CoverageFailure: + list_bibliographic = self.api.bibliographic_lookup(identifier) + if not list_bibliographic: + return self.failure(identifier, "Bibliotheca bibliographic lookup failed.") # type: ignore[no-any-return] + [bibliographic] = list_bibliographic + return self.set_bibliographic(identifier, bibliographic) # type: ignore[no-any-return] diff --git a/src/palace/manager/integration/license/bibliotheca_importer.py b/src/palace/manager/integration/license/bibliotheca/importer.py similarity index 99% rename from src/palace/manager/integration/license/bibliotheca_importer.py rename to src/palace/manager/integration/license/bibliotheca/importer.py index 3db60839b8..e638711076 100644 --- a/src/palace/manager/integration/license/bibliotheca_importer.py +++ b/src/palace/manager/integration/license/bibliotheca/importer.py @@ -11,7 +11,7 @@ from palace.manager.celery.tasks import apply from palace.manager.data_layer.policy.replacement import ReplacementPolicy -from palace.manager.integration.license.bibliotheca import BibliothecaAPI +from palace.manager.integration.license.bibliotheca.api import BibliothecaAPI from palace.manager.sqlalchemy.model.collection import Collection from palace.manager.sqlalchemy.model.coverage import Timestamp from palace.manager.sqlalchemy.model.edition import Edition diff --git a/src/palace/manager/integration/license/bibliotheca/parser.py b/src/palace/manager/integration/license/bibliotheca/parser.py new file mode 100644 index 0000000000..16fa3b9c24 --- /dev/null +++ b/src/palace/manager/integration/license/bibliotheca/parser.py @@ -0,0 +1,622 @@ +from __future__ import annotations + +import html +import itertools +import logging +import re +from abc import ABC +from collections.abc import Generator +from datetime import datetime +from typing import Literal, Optional, overload + +from frozendict import frozendict +from lxml.etree import _Element + +from palace.util.datetime_helpers import strptime_utc, to_utc +from palace.util.log import LoggerMixin + +from palace.manager.api.circulation.data import HoldInfo, LoanInfo +from palace.manager.api.circulation.exceptions import ( + AlreadyCheckedOut, + AlreadyOnHold, + CannotHold, + CannotLoan, + CurrentlyAvailable, + NoAvailableCopies, + NoLicenses, + NotCheckedOut, + NotOnHold, + PatronHoldLimitReached, + PatronLoanLimitReached, + RemoteInitiatedServerError, +) +from palace.manager.data_layer.bibliographic import BibliographicData +from palace.manager.data_layer.circulation import CirculationData +from palace.manager.data_layer.contributor import ContributorData +from palace.manager.data_layer.format import FormatData +from palace.manager.data_layer.identifier import IdentifierData +from palace.manager.data_layer.link import LinkData +from palace.manager.data_layer.measurement import MeasurementData +from palace.manager.data_layer.subject import SubjectData +from palace.manager.integration.license.bibliotheca.constants import ( + BIBLIOTHECA_SERVICE_NAME, + BIBLIOTHECA_TIME_FORMAT, +) +from palace.manager.sqlalchemy.model.circulationevent import CirculationEvent +from palace.manager.sqlalchemy.model.classification import Classification, Subject +from palace.manager.sqlalchemy.model.collection import Collection +from palace.manager.sqlalchemy.model.contributor import Contributor +from palace.manager.sqlalchemy.model.datasource import DataSource +from palace.manager.sqlalchemy.model.edition import Edition +from palace.manager.sqlalchemy.model.identifier import Identifier +from palace.manager.sqlalchemy.model.licensing import ( + DeliveryMechanism, + LicensePoolStatus, +) +from palace.manager.sqlalchemy.model.measurement import Measurement +from palace.manager.sqlalchemy.model.resource import Hyperlink, Representation +from palace.manager.util.problem_detail import BaseProblemDetailException +from palace.manager.util.xmlparser import XMLParser, XMLProcessor + + +class ItemListParser(XMLProcessor[BibliographicData], LoggerMixin): + DATE_FORMAT = "%Y-%m-%d" + YEAR_FORMAT = "%Y" + + unescape_entity_references = html.unescape + + @property + def xpath_expression(self) -> str: + return "//Item" + + parenthetical = re.compile(r" \([^)]+\)$") + + format_data_for_bibliotheca_format = frozendict( + { + "EPUB": (Representation.EPUB_MEDIA_TYPE, DeliveryMechanism.ADOBE_DRM), + "EPUB3": (Representation.EPUB_MEDIA_TYPE, DeliveryMechanism.ADOBE_DRM), + "PDF": (Representation.PDF_MEDIA_TYPE, DeliveryMechanism.ADOBE_DRM), + "MP3": (None, DeliveryMechanism.FINDAWAY_DRM), + } + ) + + @classmethod + def contributors_from_string( + cls, string: str | None, role: str = Contributor.Role.AUTHOR + ) -> list[ContributorData]: + contributors: list[ContributorData] = [] + if not string: + return contributors + + # Contributors may have two levels of entity reference escaping, + # one of which will have already been handled by the initial parse. + # We handle the potential need for a second unescaping here. + string = cls.unescape_entity_references(string) + + for sort_name in string.split(";"): + sort_name = cls.parenthetical.sub("", sort_name.strip()) + contributors.append( + ContributorData(sort_name=sort_name.strip(), roles=[role]) + ) + return contributors + + @classmethod + def parse_genre_string(self, s: str | None) -> list[SubjectData]: + genres: list[SubjectData] = [] + if not s: + return genres + for i in s.split(","): + i = i.strip() + if not i: + continue + i = ( + i.replace("&amp;", "&") + .replace("&", "&") + .replace("'", "'") + ) + genres.append( + SubjectData( + type=Subject.BISAC, + identifier=None, + name=i, + weight=Classification.TRUSTED_DISTRIBUTOR_WEIGHT, + ) + ) + return genres + + def process_one( + self, tag: _Element, namespaces: dict[str, str] | None + ) -> BibliographicData: + """Turn an tag into a BibliographicData and an encompassed CirculationData + objects, and return the BibliographicData.""" + + def value(bibliotheca_key: str) -> str | None: + return self.text_of_optional_subtag(tag, bibliotheca_key) + + primary_identifier = IdentifierData( + type=Identifier.BIBLIOTHECA_ID, identifier=value("ItemId") + ) + + identifiers = [] + for key in ("ISBN13", "PhysicalISBN"): + v = value(key) + if v: + identifiers.append(IdentifierData(type=Identifier.ISBN, identifier=v)) + + subjects = self.parse_genre_string(value("Genre")) + + title = value("Title") + subtitle = value("SubTitle") + publisher = value("Publisher") + language = value("Language") + + authors = list(self.contributors_from_string(value("Authors"))) + narrators = list( + self.contributors_from_string(value("Narrator"), Contributor.Role.NARRATOR) + ) + + published_date = None + published = value("PubDate") + if published: + formats = [self.DATE_FORMAT, self.YEAR_FORMAT] + else: + published = value("PubYear") + formats = [self.YEAR_FORMAT] + + for format in formats: + try: + published_date = strptime_utc(published, format) # type: ignore[arg-type] + except ValueError as e: + pass + + links = [] + description = value("Description") + if description: + links.append(LinkData(rel=Hyperlink.DESCRIPTION, content=description)) + + # Presume all images from Bibliotheca are JPEG. + media_type = Representation.JPEG_MEDIA_TYPE + cover_url = self.text_of_subtag(tag, "CoverLinkURL").replace("&", "&") + + # Unless the URL format has drastically changed, we should be + # able to generate a thumbnail URL based on the full-size + # cover URL found in the response document. + # + # NOTE: this is an undocumented feature of the Bibliotheca API + # which was discovered by investigating the BookLinkURL. + if "/delivery/img" in cover_url: + thumbnail_url = cover_url + "&size=NORMAL" + thumbnail = LinkData( + rel=Hyperlink.THUMBNAIL_IMAGE, href=thumbnail_url, media_type=media_type + ) + else: + thumbnail = None + cover_link = LinkData( + rel=Hyperlink.IMAGE, + href=cover_url, + media_type=media_type, + thumbnail=thumbnail, + ) + links.append(cover_link) + + alternate_url = self.text_of_subtag(tag, "BookLinkURL").replace("&", "&") + links.append(LinkData(rel="alternate", href=alternate_url)) + + measurements = [] + pages = value("NumberOfPages") + if pages: + pages_int = int(pages) + measurements.append( + MeasurementData( + quantity_measured=Measurement.PAGE_COUNT, value=pages_int + ) + ) + + circulation, medium = self._make_circulation_data(tag, primary_identifier) + + bibliographic = BibliographicData( + data_source_name=DataSource.BIBLIOTHECA, + title=title, + subtitle=subtitle, + language=language, + medium=medium, + publisher=publisher, + published=published_date, + primary_identifier_data=primary_identifier, + identifiers=identifiers, + subjects=subjects, + contributors=authors + narrators, + measurements=measurements, + links=links, + circulation=circulation, + ) + return bibliographic + + def _make_circulation_data( + self, tag: _Element, primary_identifier: IdentifierData + ) -> tuple[CirculationData, str]: + """Parse out a CirculationData containing current circulation + and formatting information. + """ + + def value(bibliotheca_key: str) -> str: + return self.text_of_subtag(tag, bibliotheca_key) + + def intvalue(key: str) -> int: + return self.int_of_subtag(tag, key) + + book_format = value("BookFormat") + medium, formats = self.internal_formats(book_format) + + licenses_owned = intvalue("TotalCopies") + try: + licenses_available = intvalue("AvailableCopies") + except IndexError: + self.log.warning( + "No information on available copies for %s", + primary_identifier.identifier, + ) + licenses_available = 0 + + patrons_in_hold_queue = intvalue("OnHoldCount") + licenses_reserved = 0 + + license_status = ( + LicensePoolStatus.ACTIVE + if licenses_owned > 0 + else LicensePoolStatus.EXHAUSTED + ) + + circulation = CirculationData( + data_source_name=DataSource.BIBLIOTHECA, + primary_identifier_data=primary_identifier, + licenses_owned=licenses_owned, + licenses_available=licenses_available, + licenses_reserved=licenses_reserved, + patrons_in_hold_queue=patrons_in_hold_queue, + formats=formats, + status=license_status, + ) + return circulation, medium + + @classmethod + def internal_formats(cls, book_format: str) -> tuple[str, list[FormatData]]: + """Convert the term Bibliotheca uses to refer to a book + format into a (medium [formats]) 2-tuple. + """ + if book_format not in cls.format_data_for_bibliotheca_format: + cls.logger().error("Unrecognized BookFormat: %s", book_format) + return Edition.BOOK_MEDIUM, [] + + content_type, drm_scheme = cls.format_data_for_bibliotheca_format[book_format] + + format = FormatData(content_type=content_type, drm_scheme=drm_scheme) + if book_format == "MP3": + medium = Edition.AUDIO_MEDIUM + else: + medium = Edition.BOOK_MEDIUM + return medium, [format] + + +class BibliothecaParser[T](XMLProcessor[T], ABC): + INPUT_TIME_FORMAT = BIBLIOTHECA_TIME_FORMAT + + @classmethod + def parse_date(cls, value: str | None) -> datetime | None: + """Parse the string Bibliotheca sends as a date. + + Usually this is a string in INPUT_TIME_FORMAT, but it might be None. + """ + if not value: + parsed = None + else: + try: + parsed = strptime_utc(value, cls.INPUT_TIME_FORMAT) + except ValueError as e: + logging.error( + 'Unable to parse Bibliotheca date: "%s"', value, exc_info=e + ) + parsed = None + return to_utc(parsed) + + @overload + def date_from_subtag(self, tag: _Element, key: str) -> datetime: ... + + @overload + def date_from_subtag( + self, tag: _Element, key: str, required: Literal[False] + ) -> datetime | None: ... + + def date_from_subtag( + self, tag: _Element, key: str, required: bool = True + ) -> datetime | None: + value = ( + self.text_of_subtag(tag, key) + if required + else self.text_of_optional_subtag(tag, key) + ) + return self.parse_date(value) + + +class ErrorParser(BibliothecaParser[BaseProblemDetailException]): + """Turn an error document from the Bibliotheca web service into a CheckoutException""" + + wrong_status = re.compile( + "the patron document status was ([^ ]+) and not one of ([^ ]+)" + ) + + loan_limit_reached = re.compile("Patron cannot loan more than [0-9]+ document") + + hold_limit_reached = re.compile("Patron cannot have more than [0-9]+ hold") + + error_mapping = { + "The patron does not have the book on hold": NotOnHold, + "The patron has no eBooks checked out": NotCheckedOut, + } + + @property + def xpath_expression(self) -> str: + return "//Error" + + def process_first(self, string: str | bytes) -> BaseProblemDetailException: + try: + return_val = super().process_first(string) + except Exception as e: + # The server sent us an error with an incorrect or + # nonstandard syntax. + if isinstance(string, bytes): + try: + debug = string.decode("utf-8") + except UnicodeDecodeError: + debug = "Unreadable error message (Unicode decode error)." + else: + debug = string + return RemoteInitiatedServerError(debug, BIBLIOTHECA_SERVICE_NAME) + + if return_val is None: + # We were not able to interpret the result as an error. + # The most likely cause is that the Bibliotheca app server is down. + return RemoteInitiatedServerError( + "Unknown error", + BIBLIOTHECA_SERVICE_NAME, + ) + + return return_val + + def process_one( + self, error_tag: _Element, namespaces: dict[str, str] | None + ) -> BaseProblemDetailException: + message = self.text_of_optional_subtag(error_tag, "Message") + if not message: + return RemoteInitiatedServerError( + "Unknown error", + BIBLIOTHECA_SERVICE_NAME, + ) + + if message in self.error_mapping: + return self.error_mapping[message](message) + if message in ("Authentication failed", "Unknown error"): + # 'Unknown error' is an unknown error on the Bibliotheca side. + # + # 'Authentication failed' could _in theory_ be an error on + # our side, but if authentication is set up improperly we + # actually get a 401 and no body. When we get a real error + # document with 'Authentication failed', it's always a + # transient error on the Bibliotheca side. Possibly some + # authentication internal to Bibliotheca has failed? Anyway, it + # happens relatively frequently. + return RemoteInitiatedServerError(message, BIBLIOTHECA_SERVICE_NAME) + + m = self.loan_limit_reached.search(message) + if m: + return PatronLoanLimitReached(message) + + m = self.hold_limit_reached.search(message) + if m: + return PatronHoldLimitReached(message) + + m = self.wrong_status.search(message) + if not m: + return RemoteInitiatedServerError(message, BIBLIOTHECA_SERVICE_NAME) + actual, expected = m.groups() + expected = expected.split(",") + + if actual == "CAN_WISH": + return NoLicenses(debug_info=message) + + if "CAN_LOAN" in expected and actual == "CAN_HOLD": + return NoAvailableCopies(debug_info=message) + + if "CAN_LOAN" in expected and actual == "HOLD": + return AlreadyOnHold(debug_info=message) + + if "CAN_LOAN" in expected and actual == "LOAN": + return AlreadyCheckedOut(debug_info=message) + + if "CAN_HOLD" in expected and actual == "CAN_LOAN": + return CurrentlyAvailable(debug_info=message) + + if "CAN_HOLD" in expected and actual == "HOLD": + return AlreadyOnHold(debug_info=message) + + if "CAN_HOLD" in expected: + return CannotHold(debug_info=message) + + if "CAN_LOAN" in expected: + return CannotLoan(debug_info=message) + + return RemoteInitiatedServerError(message, BIBLIOTHECA_SERVICE_NAME) + + +class PatronCirculationParser(XMLParser): + """Parse Bibliotheca's patron circulation status document into a list of + LoanInfo and HoldInfo objects. + """ + + id_type = Identifier.BIBLIOTHECA_ID + + def __init__(self, collection: Collection) -> None: + self.collection = collection + + def process_all(self, string: bytes | str) -> itertools.chain[LoanInfo | HoldInfo]: + xml = self._load_xml(string) + loans = self._process_all( + xml, "//Checkouts/Item", namespaces={}, handler=self.process_one_loan + ) + holds = self._process_all( + xml, "//Holds/Item", namespaces={}, handler=self.process_one_hold + ) + reserves = self._process_all( + xml, "//Reserves/Item", namespaces={}, handler=self.process_one_reserve + ) + return itertools.chain(loans, holds, reserves) + + def process_one_loan( + self, tag: _Element, namespaces: dict[str, str] + ) -> LoanInfo | None: + return self.process_one(tag, namespaces, LoanInfo) + + def process_one_hold( + self, tag: _Element, namespaces: dict[str, str] + ) -> HoldInfo | None: + return self.process_one(tag, namespaces, HoldInfo) + + def process_one_reserve( + self, tag: _Element, namespaces: dict[str, str] + ) -> HoldInfo | None: + hold_info = self.process_one(tag, namespaces, HoldInfo) + if hold_info is not None: + hold_info.hold_position = 0 + return hold_info + + def process_one[T]( + self, tag: _Element, namespaces: dict[str, str], source_class: type[T] + ) -> T | None: + if not tag.xpath("ItemId"): + # This happens for events associated with books + # no longer in our collection. + return None + + def datevalue(key: str) -> datetime: + value = self.text_of_subtag(tag, key) + return strptime_utc(value, BIBLIOTHECA_TIME_FORMAT) + + identifier = self.text_of_subtag(tag, "ItemId") + start_date = datevalue("EventStartDateInUTC") + end_date = datevalue("EventEndDateInUTC") + kwargs = { + "collection_id": self.collection.id, + "identifier_type": self.id_type, + "identifier": identifier, + "start_date": start_date, + "end_date": end_date, + } + if source_class is HoldInfo: + kwargs["hold_position"] = self.int_of_subtag(tag, "Position") + return source_class(**kwargs) + + +class DateResponseParser(BibliothecaParser[Optional[datetime]], ABC): + """Extract a date from a response.""" + + RESULT_TAG_NAME: str | None = None + DATE_TAG_NAME: str | None = None + + @property + def xpath_expression(self) -> str: + return f"/{self.RESULT_TAG_NAME}/{self.DATE_TAG_NAME}" + + def process_one( + self, tag: _Element, namespaces: dict[str, str] | None + ) -> datetime | None: + due_date = tag.text + if not due_date: + return None + return strptime_utc(due_date, EventParser.INPUT_TIME_FORMAT) + + +class CheckoutResponseParser(DateResponseParser): + """Extract due date from a checkout response.""" + + @property + def xpath_expression(self) -> str: + return f"/CheckoutResult/DueDateInUTC" + + +class HoldResponseParser(DateResponseParser): + """Extract availability date from a hold response.""" + + @property + def xpath_expression(self) -> str: + return f"/PlaceHoldResult/AvailabilityDateInUTC" + + +class EventParser( + BibliothecaParser[tuple[str, str, str | None, datetime, datetime | None, str]] +): + """Parse Bibliotheca's event file format into our native event objects.""" + + # Map Bibliotheca's event names to our names. + EVENT_NAMES = { + "CHECKOUT": CirculationEvent.DISTRIBUTOR_CHECKOUT, + "CHECKIN": CirculationEvent.DISTRIBUTOR_CHECKIN, + "HOLD": CirculationEvent.DISTRIBUTOR_HOLD_PLACE, + "RESERVED": CirculationEvent.DISTRIBUTOR_AVAILABILITY_NOTIFY, + "PURCHASE": CirculationEvent.DISTRIBUTOR_LICENSE_ADD, + "REMOVED": CirculationEvent.DISTRIBUTOR_LICENSE_REMOVE, + } + + @property + def xpath_expression(self) -> str: + return "//CloudLibraryEvent" + + def process_all( + self, string: bytes | str, no_events_error: bool = False + ) -> Generator[tuple[str, str, str | None, datetime, datetime | None, str]]: + has_events = False + # Bibliotheca occasionally returns an empty response body. An empty + # document cannot be parsed as XML (lxml raises "Document is empty" + # even in recovery mode), so treat a blank body the same as a + # response that contained no events rather than letting the parse + # error propagate as an unhandled exception. + if string.strip(): + for i in super().process_all(string): + yield i + has_events = True + + # If we are catching up on events and we expect to have a time + # period where there are no events, we don't want to consider that + # action as an error. By default, not having events is not + # considered to be an error. + if not has_events and no_events_error: + # An empty list of events may mean nothing happened, or it + # may indicate an unreported server-side error. To be + # safe, we'll treat this as a server-initiated error + # condition. If this is just a slow day, normal behavior + # will resume as soon as something happens. + raise RemoteInitiatedServerError( + "No events returned from server. This may not be an error, but treating it as one to be safe.", + BIBLIOTHECA_SERVICE_NAME, + ) + + def process_one( + self, tag: _Element, namespaces: dict[str, str] | None + ) -> tuple[str, str, str | None, datetime, datetime | None, str]: + isbn = self.text_of_subtag(tag, "ISBN") + bibliotheca_id = self.text_of_subtag(tag, "ItemId") + patron_id = self.text_of_optional_subtag(tag, "PatronId") + + start_time = self.date_from_subtag(tag, "EventStartDateTimeInUTC") + end_time = self.date_from_subtag(tag, "EventEndDateTimeInUTC", required=False) + + bibliotheca_event_type = self.text_of_subtag(tag, "EventType") + internal_event_type = self.EVENT_NAMES[bibliotheca_event_type] + + return ( + bibliotheca_id, + isbn, + patron_id, + start_time, + end_time, + internal_event_type, + ) diff --git a/src/palace/manager/integration/license/bibliotheca_purchase_record_importer.py b/src/palace/manager/integration/license/bibliotheca/purchase_record_importer.py similarity index 99% rename from src/palace/manager/integration/license/bibliotheca_purchase_record_importer.py rename to src/palace/manager/integration/license/bibliotheca/purchase_record_importer.py index 95a5ea006f..61fe56dd43 100644 --- a/src/palace/manager/integration/license/bibliotheca_purchase_record_importer.py +++ b/src/palace/manager/integration/license/bibliotheca/purchase_record_importer.py @@ -13,7 +13,7 @@ from palace.manager.celery.tasks import apply from palace.manager.data_layer.policy.replacement import ReplacementPolicy -from palace.manager.integration.license.bibliotheca import BibliothecaAPI +from palace.manager.integration.license.bibliotheca.api import BibliothecaAPI from palace.manager.sqlalchemy.model.collection import Collection from palace.manager.sqlalchemy.model.coverage import Timestamp from palace.manager.sqlalchemy.model.identifier import Identifier diff --git a/src/palace/manager/integration/license/bibliotheca_scripts.py b/src/palace/manager/integration/license/bibliotheca/scripts.py similarity index 98% rename from src/palace/manager/integration/license/bibliotheca_scripts.py rename to src/palace/manager/integration/license/bibliotheca/scripts.py index b18b20e25f..6cfa4fcc9c 100644 --- a/src/palace/manager/integration/license/bibliotheca_scripts.py +++ b/src/palace/manager/integration/license/bibliotheca/scripts.py @@ -9,7 +9,7 @@ from palace.util.exceptions import PalaceValueError from palace.manager.celery.tasks import bibliotheca -from palace.manager.integration.license.bibliotheca_purchase_record_importer import ( +from palace.manager.integration.license.bibliotheca.purchase_record_importer import ( DEFAULT_PURCHASE_RECORD_START_TIME, ) from palace.manager.scripts.base import Script diff --git a/src/palace/manager/integration/license/bibliotheca/settings.py b/src/palace/manager/integration/license/bibliotheca/settings.py new file mode 100644 index 0000000000..1fc16bf47a --- /dev/null +++ b/src/palace/manager/integration/license/bibliotheca/settings.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +from typing import Annotated + +from flask_babel import lazy_gettext as _ + +from palace.manager.api.circulation.settings import ( + BaseCirculationApiSettings, + BaseCirculationLoanSettings, +) +from palace.manager.core.config import ConfigurationAttributeValue +from palace.manager.integration.settings import ( + FormFieldType, + FormMetadata, +) + + +class BibliothecaSettings(BaseCirculationApiSettings): + username: Annotated[ + str, + FormMetadata( + label=_("Account ID"), + required=True, + ), + ] + password: Annotated[ + str, + FormMetadata( + label=_("Account Key"), + required=True, + ), + ] + external_account_id: Annotated[ + str, + FormMetadata( + label=_("Library ID"), + required=True, + ), + ] + + +class BibliothecaLibrarySettings(BaseCirculationLoanSettings): + dont_display_reserves: Annotated[ + ConfigurationAttributeValue, + FormMetadata( + label=_("Show/Hide Titles with No Available Loans"), + required=False, + description=_( + "Titles with no available loans will not be displayed in the Catalog view." + ), + type=FormFieldType.SELECT, + options={ + ConfigurationAttributeValue.YESVALUE: "Show", + ConfigurationAttributeValue.NOVALUE: "Hide", + }, + ), + ] = ConfigurationAttributeValue.YESVALUE diff --git a/src/palace/manager/scripts/availability.py b/src/palace/manager/scripts/availability.py index 9c49f5ad8b..be4f595ddf 100644 --- a/src/palace/manager/scripts/availability.py +++ b/src/palace/manager/scripts/availability.py @@ -43,13 +43,13 @@ def refresh_availability(self, identifiers: Sequence[Identifier]) -> None: collection = pool.collection if identifier.type == Identifier.BIBLIOTHECA_ID: - # Local import to avoid a circular import. bibliotheca_circulation_updater + # Local import to avoid a circular import. bibliotheca.circulation_updater # imports `palace.manager.celery.tasks`, whose package __init__ eagerly # autoloads every task module (including celery.tasks.bibliotheca), and # celery.tasks.bibliotheca imports BibliothecaCirculationUpdater back. A # top-level import here would trigger that cycle whenever availability.py is # imported before the Celery tasks have finished loading. - from palace.manager.integration.license.bibliotheca_circulation_updater import ( + from palace.manager.integration.license.bibliotheca.circulation_updater import ( BibliothecaCirculationUpdater, ) diff --git a/src/palace/manager/service/integration_registry/license_providers.py b/src/palace/manager/service/integration_registry/license_providers.py index 3c70c8eb4c..3af41abffd 100644 --- a/src/palace/manager/service/integration_registry/license_providers.py +++ b/src/palace/manager/service/integration_registry/license_providers.py @@ -16,7 +16,7 @@ class LicenseProvidersRegistry(IntegrationRegistry["CirculationApiType"]): def __init__(self) -> None: super().__init__(Goals.LICENSE_GOAL) - from palace.manager.integration.license.bibliotheca import BibliothecaAPI + from palace.manager.integration.license.bibliotheca.api import BibliothecaAPI from palace.manager.integration.license.boundless.api import BoundlessApi from palace.manager.integration.license.opds.for_distributors.api import ( OPDSForDistributorsAPI, diff --git a/tests/fixtures/database.py b/tests/fixtures/database.py index f2e5d94143..7769a573f5 100644 --- a/tests/fixtures/database.py +++ b/tests/fixtures/database.py @@ -49,10 +49,8 @@ OpdsRegistrationServiceSettings, ) from palace.manager.integration.goals import Goals -from palace.manager.integration.license.bibliotheca import ( - BibliothecaAPI, - BibliothecaSettings, -) +from palace.manager.integration.license.bibliotheca.api import BibliothecaAPI +from palace.manager.integration.license.bibliotheca.settings import BibliothecaSettings from palace.manager.integration.license.boundless.api import BoundlessApi from palace.manager.integration.license.boundless.settings import BoundlessSettings from palace.manager.integration.license.opds.for_distributors.api import ( diff --git a/tests/manager/api/circulation/test_base.py b/tests/manager/api/circulation/test_base.py index 2bcd678fe3..de538eeec5 100644 --- a/tests/manager/api/circulation/test_base.py +++ b/tests/manager/api/circulation/test_base.py @@ -7,7 +7,7 @@ from palace.manager.api.circulation.data import HoldInfo, LoanInfo from palace.manager.data_layer.format import FormatData -from palace.manager.integration.license.bibliotheca import BibliothecaAPI +from palace.manager.integration.license.bibliotheca.api import BibliothecaAPI from palace.manager.sqlalchemy.model.datasource import DataSource from palace.manager.sqlalchemy.model.identifier import Identifier from palace.manager.sqlalchemy.model.licensing import DeliveryMechanism diff --git a/tests/manager/api/controller/test_loan.py b/tests/manager/api/controller/test_loan.py index d14e3d8d90..bdcfff7c10 100644 --- a/tests/manager/api/controller/test_loan.py +++ b/tests/manager/api/controller/test_loan.py @@ -51,7 +51,7 @@ from palace.manager.core.classifier import Classifier from palace.manager.core.problem_details import INTEGRATION_ERROR, INVALID_INPUT from palace.manager.feed.acquisition import OPDSAcquisitionFeed -from palace.manager.integration.license.bibliotheca import BibliothecaAPI +from palace.manager.integration.license.bibliotheca.api import BibliothecaAPI from palace.manager.integration.license.opds.opds1.api import OPDSAPI from palace.manager.service.redis.exception import TRANSIENT_REDIS_ERRORS from palace.manager.service.redis.models.patron_activity import PatronActivity diff --git a/tests/manager/celery/tasks/test_bibliotheca.py b/tests/manager/celery/tasks/test_bibliotheca.py index fd7435c057..13015334b5 100644 --- a/tests/manager/celery/tasks/test_bibliotheca.py +++ b/tests/manager/celery/tasks/test_bibliotheca.py @@ -19,16 +19,16 @@ _circulation_update_workflow_lock, _purchase_record_workflow_lock, ) -from palace.manager.integration.license.bibliotheca import BibliothecaAPI -from palace.manager.integration.license.bibliotheca_circulation_updater import ( +from palace.manager.integration.license.bibliotheca.api import BibliothecaAPI +from palace.manager.integration.license.bibliotheca.circulation_updater import ( CIRCULATION_UPDATE_BATCH_SIZE, CIRCULATION_UPDATE_SERVICE_NAME, BatchUpdateResult, ) -from palace.manager.integration.license.bibliotheca_importer import ( +from palace.manager.integration.license.bibliotheca.importer import ( EVENT_IMPORT_SERVICE_NAME, ) -from palace.manager.integration.license.bibliotheca_purchase_record_importer import ( +from palace.manager.integration.license.bibliotheca.purchase_record_importer import ( _MARC_PAGE_SIZE, DEFAULT_PURCHASE_RECORD_START_TIME, PURCHASE_RECORD_SERVICE_NAME, @@ -130,7 +130,7 @@ def test_no_bibliotheca_collections( class TestBibliothecaImportCollection: - @patch("palace.manager.integration.license.bibliotheca_importer.BibliothecaAPI") + @patch("palace.manager.integration.license.bibliotheca.importer.BibliothecaAPI") def test_creates_timestamp_on_first_run( self, mock_api_cls: MagicMock, @@ -148,7 +148,7 @@ def test_creates_timestamp_on_first_run( assert ts is not None assert ts.finish is not None - @patch("palace.manager.integration.license.bibliotheca_importer.BibliothecaAPI") + @patch("palace.manager.integration.license.bibliotheca.importer.BibliothecaAPI") def test_starts_from_stored_timestamp( self, mock_api_cls: MagicMock, @@ -179,7 +179,7 @@ def test_starts_from_stored_timestamp( expected_start = one_hour_ago - timedelta(minutes=5) assert abs((slice_start - expected_start).total_seconds()) < 2 - @patch("palace.manager.integration.license.bibliotheca_importer.BibliothecaAPI") + @patch("palace.manager.integration.license.bibliotheca.importer.BibliothecaAPI") def test_explicit_start_used_directly( self, mock_api_cls: MagicMock, @@ -206,7 +206,7 @@ def test_explicit_start_used_directly( slice_start, _ = mock_api.get_events_between.call_args.args assert abs((slice_start - explicit_start).total_seconds()) < 1 - @patch("palace.manager.integration.license.bibliotheca_importer.BibliothecaAPI") + @patch("palace.manager.integration.license.bibliotheca.importer.BibliothecaAPI") def test_already_up_to_date( self, mock_api_cls: MagicMock, @@ -225,7 +225,7 @@ def test_already_up_to_date( mock_api_cls.return_value.get_events_between.assert_not_called() - @patch("palace.manager.integration.license.bibliotheca_importer.BibliothecaAPI") + @patch("palace.manager.integration.license.bibliotheca.importer.BibliothecaAPI") def test_replaces_when_more_slices_remain( self, mock_api_cls: MagicMock, @@ -254,7 +254,7 @@ def test_replaces_when_more_slices_remain( # The next start should be approximately 5 minutes after the slice started. assert replace_sig.kwargs["start"] is not None - @patch("palace.manager.integration.license.bibliotheca_importer.BibliothecaAPI") + @patch("palace.manager.integration.license.bibliotheca.importer.BibliothecaAPI") def test_no_replace_on_last_slice( self, mock_api_cls: MagicMock, @@ -297,7 +297,7 @@ def test_skips_when_lock_held( caplog.set_level(LogLevel.warning) with patch( - "palace.manager.integration.license.bibliotheca_importer.BibliothecaAPI" + "palace.manager.integration.license.bibliotheca.importer.BibliothecaAPI" ) as mock_api_cls: bibliotheca.import_collection.delay(collection_id=collection.id).wait() mock_api_cls.return_value.get_events_between.assert_not_called() @@ -328,7 +328,7 @@ def test_lock_not_released_on_autoretry( mock_response = MockRequestsResponse(500, content="Internal Server Error") with patch( - "palace.manager.integration.license.bibliotheca_importer.BibliothecaAPI" + "palace.manager.integration.license.bibliotheca.importer.BibliothecaAPI" ) as mock_api_cls: mock_api_cls.return_value.get_events_between.side_effect = ( BadResponseException("http://test.com", "Bad response", mock_response) @@ -413,7 +413,8 @@ def test_events_processed_end_to_end( redis_fixture: RedisFixture, ) -> None: """Smoke test: the full Celery task path processes an event and advances the - Timestamp. Detailed event-handling assertions live in test_bibliotheca_importer. + Timestamp. Detailed event-handling assertions live in + tests/manager/integration/license/bibliotheca/test_importer.py. """ collection = bibliotheca_task_fixture.collection @@ -456,7 +457,7 @@ def test_events_processed_end_to_end( assert ts.finish is not None assert ts.finish > ten_minutes_ago - @patch("palace.manager.integration.license.bibliotheca_importer.BibliothecaAPI") + @patch("palace.manager.integration.license.bibliotheca.importer.BibliothecaAPI") def test_timestamp_updated_after_each_slice( self, mock_api_cls: MagicMock, @@ -508,7 +509,7 @@ def test_multiple_collections_each_get_own_lock( lock_c1.acquire() with patch( - "palace.manager.integration.license.bibliotheca_importer.BibliothecaAPI" + "palace.manager.integration.license.bibliotheca.importer.BibliothecaAPI" ) as mock_api_cls2: mock_api_cls2.return_value.get_events_between.return_value = iter([]) # Collection 2 should process normally. @@ -646,7 +647,7 @@ def test_no_bibliotheca_collections( class TestImportPurchaseRecordsByCollection: @patch( - "palace.manager.integration.license.bibliotheca_purchase_record_importer.BibliothecaAPI" + "palace.manager.integration.license.bibliotheca.purchase_record_importer.BibliothecaAPI" ) def test_first_run_starts_from_default_start_time( self, @@ -676,7 +677,7 @@ def test_first_run_starts_from_default_start_time( assert abs((slice_start - expected_start).total_seconds()) < 1 @patch( - "palace.manager.integration.license.bibliotheca_purchase_record_importer.BibliothecaAPI" + "palace.manager.integration.license.bibliotheca.purchase_record_importer.BibliothecaAPI" ) def test_starts_from_stored_timestamp( self, @@ -709,7 +710,7 @@ def test_starts_from_stored_timestamp( assert abs((slice_start - stored_finish).total_seconds()) < 1 @patch( - "palace.manager.integration.license.bibliotheca_purchase_record_importer.BibliothecaAPI" + "palace.manager.integration.license.bibliotheca.purchase_record_importer.BibliothecaAPI" ) def test_already_up_to_date( self, @@ -731,7 +732,7 @@ def test_already_up_to_date( mock_api_cls.return_value.marc_request.assert_not_called() @patch( - "palace.manager.integration.license.bibliotheca_purchase_record_importer.BibliothecaAPI" + "palace.manager.integration.license.bibliotheca.purchase_record_importer.BibliothecaAPI" ) def test_replaces_when_more_days_remain( self, @@ -769,7 +770,7 @@ def test_replaces_when_more_days_remain( assert replace_sig.kwargs["offset"] == 1 @patch( - "palace.manager.integration.license.bibliotheca_purchase_record_importer.BibliothecaPurchaseRecordImporter.import_day" + "palace.manager.integration.license.bibliotheca.purchase_record_importer.BibliothecaPurchaseRecordImporter.import_day" ) def test_replaces_with_next_offset_when_page_full( self, @@ -814,7 +815,7 @@ def test_replaces_with_next_offset_when_page_full( assert replace_sig.kwargs["offset"] == 1 + _MARC_PAGE_SIZE @patch( - "palace.manager.integration.license.bibliotheca_purchase_record_importer.BibliothecaAPI" + "palace.manager.integration.license.bibliotheca.purchase_record_importer.BibliothecaAPI" ) def test_no_replace_on_last_day( self, @@ -860,7 +861,7 @@ def test_skips_when_lock_held( caplog.set_level(LogLevel.warning) with patch( - "palace.manager.integration.license.bibliotheca_purchase_record_importer.BibliothecaAPI" + "palace.manager.integration.license.bibliotheca.purchase_record_importer.BibliothecaAPI" ) as mock_api_cls: bibliotheca.import_purchase_records_by_collection.delay( collection_id=collection.id @@ -893,7 +894,7 @@ def test_lock_not_released_on_autoretry( mock_response = MockRequestsResponse(500, content="Internal Server Error") with patch( - "palace.manager.integration.license.bibliotheca_purchase_record_importer.BibliothecaAPI" + "palace.manager.integration.license.bibliotheca.purchase_record_importer.BibliothecaAPI" ) as mock_api_cls: mock_api_cls.return_value.marc_request.side_effect = BadResponseException( "http://test.com", "Bad response", mock_response @@ -939,7 +940,7 @@ def test_remote_initiated_server_error_retried_and_expected( ) with patch( - "palace.manager.integration.license.bibliotheca_purchase_record_importer.BibliothecaAPI" + "palace.manager.integration.license.bibliotheca.purchase_record_importer.BibliothecaAPI" ) as mock_api_cls: mock_api_cls.return_value.marc_request.side_effect = ( RemoteInitiatedServerError("boom", BibliothecaAPI.SERVICE_NAME) @@ -955,7 +956,7 @@ def test_remote_initiated_server_error_retried_and_expected( assert mock_api_cls.return_value.marc_request.call_count == 5 @patch( - "palace.manager.integration.license.bibliotheca_purchase_record_importer.BibliothecaAPI" + "palace.manager.integration.license.bibliotheca.purchase_record_importer.BibliothecaAPI" ) def test_timestamp_updated_after_each_day( self, @@ -990,7 +991,7 @@ def test_timestamp_updated_after_each_day( assert abs((ts.finish - expected_finish).total_seconds()) < 5 @patch( - "palace.manager.integration.license.bibliotheca_purchase_record_importer.BibliothecaAPI" + "palace.manager.integration.license.bibliotheca.purchase_record_importer.BibliothecaAPI" ) def test_force_reimport_clears_timestamp_before_import( self, @@ -1032,7 +1033,7 @@ def test_force_reimport_clears_timestamp_before_import( assert ts.finish == expected_finish @patch( - "palace.manager.integration.license.bibliotheca_purchase_record_importer.BibliothecaAPI" + "palace.manager.integration.license.bibliotheca.purchase_record_importer.BibliothecaAPI" ) def test_reset_timestamp_not_forwarded_to_replacement( self, @@ -1085,7 +1086,7 @@ def test_chain_runs_through_multiple_days_end_to_end( ) with patch( - "palace.manager.integration.license.bibliotheca_purchase_record_importer.BibliothecaAPI" + "palace.manager.integration.license.bibliotheca.purchase_record_importer.BibliothecaAPI" ) as mock_api_cls: # Return a fresh empty iterator on every call so each day's page # returns 0 records (day_complete=True, no pagination within a day). @@ -1165,7 +1166,7 @@ def test_stops_chain_gracefully_when_collection_marked_for_deletion( caplog.set_level(LogLevel.warning) with patch( - "palace.manager.integration.license.bibliotheca_purchase_record_importer.BibliothecaAPI" + "palace.manager.integration.license.bibliotheca.purchase_record_importer.BibliothecaAPI" ) as mock_api_cls: bibliotheca.import_purchase_records_by_collection.delay( collection_id=collection.id, @@ -1312,7 +1313,7 @@ def test_first_invocation_reads_offset_from_timestamp( mock_updater.update_batch.assert_called_once_with(99) @patch( - "palace.manager.integration.license.bibliotheca_circulation_updater.BibliothecaAPI" + "palace.manager.integration.license.bibliotheca.circulation_updater.BibliothecaAPI" ) def test_already_complete_no_replace( self, @@ -1336,7 +1337,7 @@ def test_already_complete_no_replace( mock_replace.assert_not_called() @patch( - "palace.manager.integration.license.bibliotheca_circulation_updater.BibliothecaAPI" + "palace.manager.integration.license.bibliotheca.circulation_updater.BibliothecaAPI" ) def test_full_batch_issues_replace_with_next_offset( self, @@ -1373,7 +1374,7 @@ def test_full_batch_issues_replace_with_next_offset( assert replace_sig.kwargs["offset"] == 100 @patch( - "palace.manager.integration.license.bibliotheca_circulation_updater.BibliothecaAPI" + "palace.manager.integration.license.bibliotheca.circulation_updater.BibliothecaAPI" ) def test_partial_batch_no_replace( self, @@ -1406,7 +1407,7 @@ def test_partial_batch_no_replace( mock_replace.assert_not_called() @patch( - "palace.manager.integration.license.bibliotheca_circulation_updater.BibliothecaAPI" + "palace.manager.integration.license.bibliotheca.circulation_updater.BibliothecaAPI" ) def test_replace_carries_collection_id( self, @@ -1643,7 +1644,7 @@ def test_stops_chain_gracefully_when_collection_marked_for_deletion( assert collection.name in caplog.text @patch( - "palace.manager.integration.license.bibliotheca_circulation_updater.BibliothecaAPI" + "palace.manager.integration.license.bibliotheca.circulation_updater.BibliothecaAPI" ) def test_full_sweep_chains_through_every_batch_and_releases_lock( self, diff --git a/tests/manager/celery/tasks/test_playtime_entries.py b/tests/manager/celery/tasks/test_playtime_entries.py index a88bb91498..914fe0ec44 100644 --- a/tests/manager/celery/tasks/test_playtime_entries.py +++ b/tests/manager/celery/tasks/test_playtime_entries.py @@ -20,7 +20,7 @@ sum_playtime_entries, ) from palace.manager.core.config import Configuration -from palace.manager.integration.license.bibliotheca import BibliothecaAPI +from palace.manager.integration.license.bibliotheca.api import BibliothecaAPI from palace.manager.integration.license.opds.for_distributors.api import ( OPDSForDistributorsAPI, ) diff --git a/tests/manager/core/test_monitor.py b/tests/manager/core/test_monitor.py index 693b35c0bf..80848c6dcc 100644 --- a/tests/manager/core/test_monitor.py +++ b/tests/manager/core/test_monitor.py @@ -17,7 +17,7 @@ TimelineMonitor, TimestampData, ) -from palace.manager.integration.license.bibliotheca import BibliothecaAPI +from palace.manager.integration.license.bibliotheca.api import BibliothecaAPI from palace.manager.integration.license.opds.opds1.api import OPDSAPI from palace.manager.integration.license.overdrive.api import OverdriveAPI from palace.manager.service import container diff --git a/tests/manager/feed/worklist/test_base.py b/tests/manager/feed/worklist/test_base.py index 45bf9fb1a6..53f63cb964 100644 --- a/tests/manager/feed/worklist/test_base.py +++ b/tests/manager/feed/worklist/test_base.py @@ -17,7 +17,7 @@ from palace.manager.feed.facets.search import SearchFacets from palace.manager.feed.worklist.base import WorkList from palace.manager.feed.worklist.top_level import TopLevelWorkList -from palace.manager.integration.license.bibliotheca import BibliothecaAPI +from palace.manager.integration.license.bibliotheca.api import BibliothecaAPI from palace.manager.integration.license.opds.odl.api import OPDS2WithODLApi from palace.manager.search.external_search import ExternalSearchIndex from palace.manager.search.filter import Filter diff --git a/tests/manager/integration/license/bibliotheca/__init__.py b/tests/manager/integration/license/bibliotheca/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/manager/integration/license/bibliotheca/conftest.py b/tests/manager/integration/license/bibliotheca/conftest.py new file mode 100644 index 0000000000..61d94c05aa --- /dev/null +++ b/tests/manager/integration/license/bibliotheca/conftest.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +import pytest + +from tests.fixtures.database import DatabaseTransactionFixture +from tests.fixtures.files import BibliothecaFilesFixture +from tests.mocks.bibliotheca import MockBibliothecaAPI + + +class BibliothecaAPITestFixture: + def __init__( + self, + db: DatabaseTransactionFixture, + files: BibliothecaFilesFixture, + ): + self.files = files + self.db = db + self.collection = MockBibliothecaAPI.mock_collection( + db.session, db.default_library() + ) + self.api = MockBibliothecaAPI(db.session, self.collection) + + +@pytest.fixture(scope="function") +def bibliotheca_fixture( + db: DatabaseTransactionFixture, + bibliotheca_files_fixture: BibliothecaFilesFixture, +) -> BibliothecaAPITestFixture: + return BibliothecaAPITestFixture(db, bibliotheca_files_fixture) diff --git a/tests/manager/integration/license/test_bibliotheca.py b/tests/manager/integration/license/bibliotheca/test_api.py similarity index 51% rename from tests/manager/integration/license/test_bibliotheca.py rename to tests/manager/integration/license/bibliotheca/test_api.py index 1f9e5441d3..a87a7d9570 100644 --- a/tests/manager/integration/license/test_bibliotheca.py +++ b/tests/manager/integration/license/bibliotheca/test_api.py @@ -2,8 +2,7 @@ import json import random -from datetime import date, timedelta -from typing import TYPE_CHECKING, cast +from datetime import timedelta from unittest.mock import create_autospec, patch import pytest @@ -11,82 +10,32 @@ from palace.util.datetime_helpers import datetime_utc, utc_now -from palace.manager.api.circulation.data import HoldInfo, LoanInfo from palace.manager.api.circulation.dispatcher import CirculationApiDispatcher from palace.manager.api.circulation.exceptions import ( - AlreadyCheckedOut, - AlreadyOnHold, - CannotHold, - CirculationException, - CurrentlyAvailable, - NoAvailableCopies, - NoLicenses, - NotCheckedOut, - NotOnHold, PatronHoldLimitReached, - PatronLoanLimitReached, RemoteInitiatedServerError, ) from palace.manager.api.circulation.fulfillment import Fulfillment from palace.manager.api.web_publication_manifest import FindawayManifest from palace.manager.celery.tasks import apply -from palace.manager.integration.license.bibliotheca import ( - BibliothecaAPI, - BibliothecaBibliographicCoverageProvider, - BibliothecaParser, - CheckoutResponseParser, - ErrorParser, - EventParser, - ItemListParser, - PatronCirculationParser, -) -from palace.manager.scripts.coverage_provider import RunCollectionCoverageProviderScript -from palace.manager.sqlalchemy.model.circulationevent import CirculationEvent -from palace.manager.sqlalchemy.model.classification import Subject -from palace.manager.sqlalchemy.model.contributor import Contributor +from palace.manager.integration.license.bibliotheca.api import BibliothecaAPI from palace.manager.sqlalchemy.model.datasource import DataSource -from palace.manager.sqlalchemy.model.edition import Edition from palace.manager.sqlalchemy.model.identifier import Identifier from palace.manager.sqlalchemy.model.licensing import ( DeliveryMechanism, LicensePoolDeliveryMechanism, - LicensePoolStatus, ) -from palace.manager.sqlalchemy.model.measurement import Measurement -from palace.manager.sqlalchemy.model.resource import Hyperlink, Representation +from palace.manager.sqlalchemy.model.resource import Representation from palace.manager.util.http.exception import ( BadResponseException, RemoteIntegrationException, ) from palace.manager.util.web_publication_manifest import AudiobookManifest +from tests.manager.integration.license.bibliotheca.conftest import ( + BibliothecaAPITestFixture, +) from tests.mocks.bibliotheca import MockBibliothecaAPI -if TYPE_CHECKING: - from tests.fixtures.database import DatabaseTransactionFixture - from tests.fixtures.files import BibliothecaFilesFixture - - -class BibliothecaAPITestFixture: - def __init__( - self, - db: DatabaseTransactionFixture, - files: BibliothecaFilesFixture, - ): - self.files = files - self.db = db - self.collection = MockBibliothecaAPI.mock_collection( - db.session, db.default_library() - ) - self.api = MockBibliothecaAPI(db.session, self.collection) - - -@pytest.fixture(scope="function") -def bibliotheca_fixture( - db: DatabaseTransactionFixture, - bibliotheca_files_fixture: BibliothecaFilesFixture, -) -> BibliothecaAPITestFixture: - return BibliothecaAPITestFixture(db, bibliotheca_files_fixture) - class TestBibliothecaAPI: def test__run_self_tests( @@ -709,605 +658,3 @@ def test_findaway_license_to_webpub_manifest( # The total duration, in seconds, has been added to metadata. assert 28371 == int(metadata["duration"]) - - -# Tests of the various parser classes. -# - - -class TestBibliothecaParser: - def test_parse_date(self, bibliotheca_fixture: BibliothecaAPITestFixture): - v = BibliothecaParser.parse_date("2016-01-02T12:34:56") - assert v == datetime_utc(2016, 1, 2, 12, 34, 56) - - assert BibliothecaParser.parse_date(None) is None - assert BibliothecaParser.parse_date("Some weird value") is None - - -class TestEventParser: - def test_parse_empty_list(self, bibliotheca_fixture: BibliothecaAPITestFixture): - data = bibliotheca_fixture.files.sample_data("empty_event_batch.xml") - - # By default, we consider an empty batch of events not - # as an error. - events = list(EventParser().process_all(data)) - assert [] == events - - # But if we consider not having events for a certain time - # period, then an exception should be raised. - no_events_error = True - with pytest.raises(RemoteInitiatedServerError) as excinfo: - list(EventParser().process_all(data, no_events_error)) - assert ( - "No events returned from server. This may not be an error, but treating it as one to be safe." - in str(excinfo.value) - ) - - @pytest.mark.parametrize("data", [b"", b" \n ", ""]) - def test_parse_empty_response_body(self, data: bytes | str): - # Bibliotheca occasionally returns a completely empty response - # body, which cannot be parsed as XML. We treat it the same as a - # response containing no events rather than raising a parse error. - events = list(EventParser().process_all(data)) - assert [] == events - - # And, as with a well-formed empty batch, we raise when the caller - # has indicated that having no events should be treated as an error. - with pytest.raises(RemoteInitiatedServerError) as excinfo: - list(EventParser().process_all(data, no_events_error=True)) - assert ( - "No events returned from server. This may not be an error, but treating it as one to be safe." - in str(excinfo.value) - ) - - def test_parse_empty_end_date_event( - self, bibliotheca_fixture: BibliothecaAPITestFixture - ): - data = bibliotheca_fixture.files.sample_data("empty_end_date_event.xml") - [event] = list(EventParser().process_all(data)) - (threem_id, isbn, patron_id, start_time, end_time, internal_event_type) = event - assert "d5rf89" == threem_id - assert "9781101190623" == isbn - assert None == patron_id - assert datetime_utc(2016, 4, 28, 11, 4, 6) == start_time - assert None == end_time - assert "distributor_license_add" == internal_event_type - - -class TestPatronCirculationParser: - def test_parse(self, bibliotheca_fixture: BibliothecaAPITestFixture): - data = bibliotheca_fixture.files.sample_data("checkouts.xml") - collection = bibliotheca_fixture.collection - loans_and_holds = list(PatronCirculationParser(collection).process_all(data)) - loans = [x for x in loans_and_holds if isinstance(x, LoanInfo)] - holds = [x for x in loans_and_holds if isinstance(x, HoldInfo)] - assert 2 == len(loans) - assert 2 == len(holds) - [l1, l2] = sorted(loans, key=lambda x: str(x.identifier)) - assert "1ad589" == l1.identifier - assert "cgaxr9" == l2.identifier - expect_loan_start = datetime_utc(2015, 3, 20, 18, 50, 22) - expect_loan_end = datetime_utc(2015, 4, 10, 18, 50, 22) - assert expect_loan_start == l1.start_date - assert expect_loan_end == l1.end_date - - [h1, h2] = sorted(holds, key=lambda x: str(x.identifier)) - - # This is the book on reserve. - assert collection.id == h1.collection_id - assert "9wd8" == h1.identifier - expect_hold_start = datetime_utc(2015, 5, 25, 17, 5, 34) - expect_hold_end = datetime_utc(2015, 5, 27, 17, 5, 34) - assert expect_hold_start == h1.start_date - assert expect_hold_end == h1.end_date - assert 0 == h1.hold_position - - # This is the book on hold. - assert "d4o8r9" == h2.identifier - assert collection.id == h2.collection_id - expect_hold_start = datetime_utc(2015, 3, 24, 15, 6, 56) - expect_hold_end = datetime_utc(2015, 3, 24, 15, 7, 51) - assert expect_hold_start == h2.start_date - assert expect_hold_end == h2.end_date - assert 4 == h2.hold_position - - -class TestCheckoutResponseParser: - def test_parse(self, bibliotheca_fixture: BibliothecaAPITestFixture): - data = bibliotheca_fixture.files.sample_data("successful_checkout.xml") - due_date = CheckoutResponseParser().process_first(data) - assert datetime_utc(2015, 4, 16, 0, 32, 36) == due_date - - -class TestErrorParser: - BIBLIOTHECA_ERROR_RESPONSE_BODY_TEMPLATE = ( - '' - "Gen-001" - "{message}" - "" - ) - - @pytest.mark.parametrize( - "incoming_message, error_class, message, debug_message", - [ - ( - "Patron cannot loan more than 12 documents", - PatronLoanLimitReached, - "Patron cannot loan more than 12 documents", - None, - ), - ( - "Patron cannot have more than 15 holds", - PatronHoldLimitReached, - "Patron cannot have more than 15 holds", - None, - ), - ( - "the patron document status was CAN_WISH and not one of CAN_LOAN,RESERVATION", - NoLicenses, - "The library currently has no licenses for this book.", - "the patron document status was CAN_WISH and not one of CAN_LOAN,RESERVATION", - ), - ( - "the patron document status was CAN_HOLD and not one of CAN_LOAN,RESERVATION", - NoAvailableCopies, - "No copies available to check out.", - "the patron document status was CAN_HOLD and not one of CAN_LOAN,RESERVATION", - ), - ( - "the patron document status was LOAN and not one of CAN_LOAN,RESERVATION", - AlreadyCheckedOut, - "You already have this book checked out.", - "the patron document status was LOAN and not one of CAN_LOAN,RESERVATION", - ), - ( - "The patron has no eBooks checked out", - NotCheckedOut, - "The patron has no eBooks checked out", - None, - ), - ( - "the patron document status was CAN_LOAN and not one of CAN_HOLD", - CurrentlyAvailable, - "Cannot place a hold on an available title.", - "the patron document status was CAN_LOAN and not one of CAN_HOLD", - ), - ( - "the patron document status was HOLD and not one of CAN_HOLD", - AlreadyOnHold, - "You already have this book on hold.", - "the patron document status was HOLD and not one of CAN_HOLD", - ), - ( - "The patron does not have the book on hold", - NotOnHold, - "The patron does not have the book on hold", - None, - ), - # This is such a weird case we don't have a special exception for it. - ( - "the patron document status was LOAN and not one of CAN_HOLD", - CannotHold, - "Could not place hold (reason unknown).", - "the patron document status was LOAN and not one of CAN_HOLD", - ), - ], - ) - def test_exception( - self, - incoming_message: str, - error_class: type[CirculationException], - message: str, - debug_message: str | None, - ): - document = self.BIBLIOTHECA_ERROR_RESPONSE_BODY_TEMPLATE.format( - message=incoming_message - ) - error = ErrorParser().process_first(document) - assert error.__class__ is error_class - assert error.problem_detail.detail == message - assert error.problem_detail.debug_message == debug_message - - @pytest.mark.parametrize( - "incoming_message, incoming_message_from_file, error_string", - [ - ( - # Simulate the message we get when the server goes down. - "The server has encountered an error", - None, - "The server has encountered an error", - ), - ( - # Simulate an unexpected response, which is not a unicode string. - b"Beep boop bytes", - None, - "Beep boop bytes", - ), - ( - # Simulate an unexpected response, which cannot be decoded as a string. - b"\xde\xad\xbe\xef", - None, - "Unreadable error message (Unicode decode error).", - ), - ( - # Simulate the message we get when the server gives a vague error. - None, - "error_unknown.xml", - "Unknown error", - ), - ( - # Simulate the message we get when the error message is - # 'Authentication failed' but our authentication information is - # set up correctly. - None, - "error_authentication_failed.xml", - "Authentication failed", - ), - ( - """This error does not follow the standard set out by Bibliotheca.""", - None, - "Unknown error", - ), - ( - # Empty error message - """""", - None, - "Unknown error", - ), - ], - ) - def test_remote_initiated_server_error( - self, - incoming_message: str | bytes | None, - incoming_message_from_file: str | None, - error_string: str, - bibliotheca_files_fixture: BibliothecaFilesFixture, - ): - if incoming_message_from_file: - incoming_message = bibliotheca_files_fixture.sample_text( - incoming_message_from_file - ) - assert incoming_message is not None - error = ErrorParser().process_first(incoming_message) - assert isinstance(error, RemoteInitiatedServerError) - - assert BibliothecaAPI.SERVICE_NAME == error.service_name - assert error_string == str(error) - - problem = error.problem_detail - assert 502 == problem.status_code - assert "Integration error communicating with Bibliotheca" == problem.detail - assert "Third-party service failed." == problem.title - - -class TestBibliothecaEventParser: - # Sample event feed to test out the parser. - TWO_EVENTS = """ - 1b0d6667-a10e-424a-9f73-fb6f6d41308e - 2014-04-14T13:59:05.6920303Z - 2014-04-03T00:00:34 - - - test-library - event-1 - CHECKIN - 2014-04-03T00:00:23 - 2014-04-03T00:00:23 - theitem1 - 900isbn1 - patronid1 - 2014-04-14T13:59:05 - - - test-library - event-2 - CHECKOUT - 2014-04-03T00:00:34 - 2014-04-02T23:57:37 - theitem2 - 900isbn2 - patronid2 - 2014-04-14T13:59:05 - - - -""" - - def test_parse_event_batch(self): - # Parsing the XML gives us two events. - event1, event2 = EventParser().process_all(self.TWO_EVENTS) - - (threem_id, isbn, patron_id, start_time, end_time, internal_event_type) = event1 - - assert "theitem1" == threem_id - assert "900isbn1" == isbn - assert "patronid1" == patron_id - assert CirculationEvent.DISTRIBUTOR_CHECKIN == internal_event_type - assert start_time == end_time - - (threem_id, isbn, patron_id, start_time, end_time, internal_event_type) = event2 - assert "theitem2" == threem_id - assert "900isbn2" == isbn - assert "patronid2" == patron_id - assert CirculationEvent.DISTRIBUTOR_CHECKOUT == internal_event_type - - # Verify that start and end time were parsed correctly. - correct_start = datetime_utc(2014, 4, 3, 0, 0, 34) - correct_end = datetime_utc(2014, 4, 2, 23, 57, 37) - assert correct_start == start_time - assert correct_end == end_time - - -class TestItemListParser: - def test_contributors_for_string(cls): - authors = list( - ItemListParser.contributors_from_string( - "Walsh, Jill Paton; Sayers, Dorothy L." - ) - ) - assert [x.sort_name for x in authors] == [ - "Walsh, Jill Paton", - "Sayers, Dorothy L.", - ] - assert [x.roles for x in authors] == [ - (Contributor.Role.AUTHOR,), - (Contributor.Role.AUTHOR,), - ] - - # Parentheticals are stripped. - [author] = ItemListParser.contributors_from_string( - "Baum, Frank L. (Frank Lyell)" - ) - assert "Baum, Frank L." == author.sort_name - - # Contributors may have two levels of entity reference escaping, - # one of which will have already been handled by the initial parse. - # So, we'll test zero and one escapings here. - authors = list( - ItemListParser.contributors_from_string( - "Raji Codell, Esmé; Raji Codell, Esmé" - ) - ) - author_names = [a.sort_name for a in authors] - assert len(authors) == 2 - assert len(set(author_names)) == 1 - assert all("Raji Codell, Esmé" == name for name in author_names) - - # It's possible to specify some role other than AUTHOR_ROLE. - narrators = list( - ItemListParser.contributors_from_string( - "Callow, Simon; Mann, Bruce; Hagon, Garrick", Contributor.Role.NARRATOR - ) - ) - for narrator in narrators: - assert (Contributor.Role.NARRATOR,) == narrator.roles - assert ["Callow, Simon", "Mann, Bruce", "Hagon, Garrick"] == [ - narrator.sort_name for narrator in narrators - ] - - def test_parse_genre_string(self): - def f(genre_string): - genres = ItemListParser.parse_genre_string(genre_string) - assert all([x.type == Subject.BISAC for x in genres]) - return [x.name for x in genres] - - assert ["Children's Health", "Health"] == f("Children&#39;s Health,Health,") - - assert [ - "Action & Adventure", - "Science Fiction", - "Fantasy", - "Magic", - "Renaissance", - ] == f( - "Action &amp; Adventure,Science Fiction, Fantasy, Magic,Renaissance," - ) - - def test_item_list(self, bibliotheca_fixture: BibliothecaAPITestFixture): - data = bibliotheca_fixture.files.sample_data("item_metadata_list_mini.xml") - data_parsed = list(ItemListParser().process_all(data)) - - # There should be 2 items in the list. - assert 2 == len(data_parsed) - - cooked = data_parsed[0] - - assert "The Incense Game" == cooked.title - assert "A Novel of Feudal Japan" == cooked.subtitle - assert Edition.BOOK_MEDIUM == cooked.medium - assert "eng" == cooked.language - assert "St. Martin's Press" == cooked.publisher - assert date(year=2012, month=9, day=17) == cooked.published - - primary = cooked.primary_identifier_data - assert "ddf4gr9" == primary.identifier - assert Identifier.BIBLIOTHECA_ID == primary.type - - identifiers = sorted(cooked.identifiers, key=lambda x: x.identifier) - assert ["9781250015280", "9781250031112", "ddf4gr9"] == [ - x.identifier for x in identifiers - ] - - [author] = cooked.contributors - assert "Rowland, Laura Joh" == author.sort_name - assert (Contributor.Role.AUTHOR,) == author.roles - - subjects = [x.name for x in cooked.subjects if x.name is not None] - assert ["Children's Health", "Mystery & Detective"] == sorted(subjects) - - [pages] = cooked.measurements - assert Measurement.PAGE_COUNT == pages.quantity_measured - assert 304 == pages.value - - [alternate, image, description] = sorted(cooked.links, key=lambda x: x.rel) - assert "alternate" == alternate.rel - assert alternate.href.startswith("http://ebook.3m.com/library") - - # We have a full-size image... - assert Hyperlink.IMAGE == image.rel - assert Representation.JPEG_MEDIA_TYPE == image.media_type - assert image.href is not None - assert image.href.startswith("http://ebook.3m.com/delivery") - assert "documentID=ddf4gr9" in image.href - assert "&size=NORMAL" not in image.href - - # ... and a thumbnail, which we obtained by adding an argument - # to the main image URL. - thumbnail = image.thumbnail - assert Hyperlink.THUMBNAIL_IMAGE == thumbnail.rel - assert Representation.JPEG_MEDIA_TYPE == thumbnail.media_type - assert thumbnail.href == image.href + "&size=NORMAL" - - # We have a description. - assert Hyperlink.DESCRIPTION == description.rel - assert isinstance(description.content, str) - assert description.content.startswith("Winner") - - def test_multiple_contributor_roles( - self, bibliotheca_fixture: BibliothecaAPITestFixture - ): - data = bibliotheca_fixture.files.sample_data("item_metadata_audio.xml") - [parsed_data] = list(ItemListParser().process_all(data)) - names_and_roles = [] - for c in parsed_data.contributors: - [role] = c.roles - names_and_roles.append((c.sort_name, role)) - - # We found one author and three narrators. - assert sorted( - [ - ("Riggs, Ransom", "Author"), - ("Callow, Simon", "Narrator"), - ("Mann, Bruce", "Narrator"), - ("Hagon, Garrick", "Narrator"), - ] - ) == sorted(names_and_roles) - - def test_circulation_data_status( - self, bibliotheca_fixture: BibliothecaAPITestFixture - ): - """Test that CirculationData from ItemListParser has correct status.""" - data = bibliotheca_fixture.files.sample_data("item_metadata_list_mini.xml") - data_parsed = list(ItemListParser().process_all(data)) - - # Check the first book's circulation data - bibliographic1 = data_parsed[0] - circulation1 = bibliographic1.circulation - - # This book has 1 license, so status should be ACTIVE - assert circulation1.licenses_owned == 1 - assert circulation1.licenses_available == 1 - assert circulation1.status == LicensePoolStatus.ACTIVE - - # Check the second book's circulation data - bibliographic2 = data_parsed[1] - circulation2 = bibliographic2.circulation - - # This book also has licenses, so status should be ACTIVE - assert circulation2.licenses_owned == 1 - assert circulation2.status == LicensePoolStatus.ACTIVE - - def test_circulation_data_status_exhausted( - self, bibliotheca_fixture: BibliothecaAPITestFixture - ): - """Test that CirculationData has EXHAUSTED status when licenses_owned is 0.""" - data = bibliotheca_fixture.files.sample_data("item_metadata_list_mini.xml") - # Replace TotalCopies with 0 to test EXHAUSTED status - data = data.replace( - b"1", b"0" - ) - - data_parsed = list(ItemListParser().process_all(data)) - - # Both books should have EXHAUSTED status - for bibliographic in data_parsed: - circulation = bibliographic.circulation - assert circulation.licenses_owned == 0 - assert circulation.status == LicensePoolStatus.EXHAUSTED - - -class TestBibliographicCoverageProvider(TestBibliothecaAPI): - """Test the code that looks up bibliographic information from Bibliotheca.""" - - def test_script_instantiation(self, bibliotheca_fixture: BibliothecaAPITestFixture): - """Test that RunCollectionCoverageProviderScript can instantiate - this coverage provider. - """ - script = RunCollectionCoverageProviderScript( - BibliothecaBibliographicCoverageProvider, - bibliotheca_fixture.db.session, - api_class=MockBibliothecaAPI, - ) - [provider] = script.providers - assert isinstance(provider, BibliothecaBibliographicCoverageProvider) - assert isinstance(provider.api, MockBibliothecaAPI) - - def test_process_item_creates_presentation_ready_work( - self, bibliotheca_fixture: BibliothecaAPITestFixture - ): - db = bibliotheca_fixture.db - # Test the normal workflow where we ask Bibliotheca for data, - # Bibliotheca provides it, and we create a presentation-ready work. - identifier = db.identifier(identifier_type=Identifier.BIBLIOTHECA_ID) - identifier.identifier = "ddf4gr9" - - # This book has no LicensePools. - assert [] == identifier.licensed_through - - # Run it through the BibliothecaBibliographicCoverageProvider - provider = BibliothecaBibliographicCoverageProvider( - bibliotheca_fixture.collection, api_class=MockBibliothecaAPI - ) - api = cast(MockBibliothecaAPI, provider.api) - data = bibliotheca_fixture.files.sample_data("item_metadata_single.xml") - - # We can't use bibliotheca_fixture.api because that's not the same object - # as the one created by the coverage provider. - api.queue_response(200, content=data) - [result] = provider.process_batch([identifier]) - assert identifier == result - # A LicensePool was created and populated with format and availability - # information. - [pool] = identifier.licensed_through - assert 1 == pool.licenses_owned - assert 1 == pool.licenses_available - [lpdm] = pool.delivery_mechanisms - assert ( - "application/epub+zip (application/vnd.adobe.adept+xml)" - == lpdm.delivery_mechanism.name - ) - - # A Work was created and made presentation ready. - assert "The Incense Game" == pool.work.title - assert True == pool.work.presentation_ready - - def test_internal_formats(self): - m = ItemListParser.internal_formats - - def _check_format(input, expect_medium, expect_format, expect_drm): - medium, formats = m(input) - assert medium == expect_medium - [format] = formats - assert expect_format == format.content_type - assert expect_drm == format.drm_scheme - - rep = Representation - adobe = DeliveryMechanism.ADOBE_DRM - findaway = DeliveryMechanism.FINDAWAY_DRM - book = Edition.BOOK_MEDIUM - - # Verify that we handle the known strings from Bibliotheca - # appropriately. - _check_format("EPUB", book, rep.EPUB_MEDIA_TYPE, adobe) - _check_format("EPUB3", book, rep.EPUB_MEDIA_TYPE, adobe) - _check_format("PDF", book, rep.PDF_MEDIA_TYPE, adobe) - _check_format("MP3", Edition.AUDIO_MEDIUM, None, findaway) - - # Now Try a string we don't recognize from Bibliotheca. - medium, formats = m("Unknown") - - # We assume it's a book. - assert Edition.BOOK_MEDIUM == medium - - # But we don't know which format. - assert [] == formats diff --git a/tests/manager/integration/license/test_bibliotheca_circulation_updater.py b/tests/manager/integration/license/bibliotheca/test_circulation_updater.py similarity index 99% rename from tests/manager/integration/license/test_bibliotheca_circulation_updater.py rename to tests/manager/integration/license/bibliotheca/test_circulation_updater.py index 4d19d4812d..8999e728ff 100644 --- a/tests/manager/integration/license/test_bibliotheca_circulation_updater.py +++ b/tests/manager/integration/license/bibliotheca/test_circulation_updater.py @@ -11,8 +11,8 @@ from palace.manager.data_layer.bibliographic import BibliographicData from palace.manager.data_layer.circulation import CirculationData from palace.manager.data_layer.identifier import IdentifierData -from palace.manager.integration.license.bibliotheca import BibliothecaAPI -from palace.manager.integration.license.bibliotheca_circulation_updater import ( +from palace.manager.integration.license.bibliotheca.api import BibliothecaAPI +from palace.manager.integration.license.bibliotheca.circulation_updater import ( CIRCULATION_UPDATE_BATCH_SIZE, CIRCULATION_UPDATE_SERVICE_NAME, BibliothecaCirculationUpdater, diff --git a/tests/manager/integration/license/bibliotheca/test_coverage.py b/tests/manager/integration/license/bibliotheca/test_coverage.py new file mode 100644 index 0000000000..542c263d9c --- /dev/null +++ b/tests/manager/integration/license/bibliotheca/test_coverage.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +from typing import cast + +from palace.manager.integration.license.bibliotheca.coverage import ( + BibliothecaBibliographicCoverageProvider, +) +from palace.manager.scripts.coverage_provider import RunCollectionCoverageProviderScript +from palace.manager.sqlalchemy.model.identifier import Identifier +from tests.manager.integration.license.bibliotheca.conftest import ( + BibliothecaAPITestFixture, +) +from tests.mocks.bibliotheca import MockBibliothecaAPI + + +class TestBibliographicCoverageProvider: + """Test the code that looks up bibliographic information from Bibliotheca.""" + + def test_script_instantiation(self, bibliotheca_fixture: BibliothecaAPITestFixture): + """Test that RunCollectionCoverageProviderScript can instantiate + this coverage provider. + """ + script = RunCollectionCoverageProviderScript( + BibliothecaBibliographicCoverageProvider, + bibliotheca_fixture.db.session, + api_class=MockBibliothecaAPI, + ) + [provider] = script.providers + assert isinstance(provider, BibliothecaBibliographicCoverageProvider) + assert isinstance(provider.api, MockBibliothecaAPI) + + def test_process_item_creates_presentation_ready_work( + self, bibliotheca_fixture: BibliothecaAPITestFixture + ): + db = bibliotheca_fixture.db + # Test the normal workflow where we ask Bibliotheca for data, + # Bibliotheca provides it, and we create a presentation-ready work. + identifier = db.identifier(identifier_type=Identifier.BIBLIOTHECA_ID) + identifier.identifier = "ddf4gr9" + + # This book has no LicensePools. + assert [] == identifier.licensed_through + + # Run it through the BibliothecaBibliographicCoverageProvider + provider = BibliothecaBibliographicCoverageProvider( + bibliotheca_fixture.collection, api_class=MockBibliothecaAPI + ) + api = cast(MockBibliothecaAPI, provider.api) + data = bibliotheca_fixture.files.sample_data("item_metadata_single.xml") + + # We can't use bibliotheca_fixture.api because that's not the same object + # as the one created by the coverage provider. + api.queue_response(200, content=data) + [result] = provider.process_batch([identifier]) + assert identifier == result + # A LicensePool was created and populated with format and availability + # information. + [pool] = identifier.licensed_through + assert 1 == pool.licenses_owned + assert 1 == pool.licenses_available + [lpdm] = pool.delivery_mechanisms + assert ( + "application/epub+zip (application/vnd.adobe.adept+xml)" + == lpdm.delivery_mechanism.name + ) + + # A Work was created and made presentation ready. + assert "The Incense Game" == pool.work.title + assert True == pool.work.presentation_ready diff --git a/tests/manager/integration/license/test_bibliotheca_importer.py b/tests/manager/integration/license/bibliotheca/test_importer.py similarity index 98% rename from tests/manager/integration/license/test_bibliotheca_importer.py rename to tests/manager/integration/license/bibliotheca/test_importer.py index 057ab483ec..dc651a8bee 100644 --- a/tests/manager/integration/license/test_bibliotheca_importer.py +++ b/tests/manager/integration/license/bibliotheca/test_importer.py @@ -8,8 +8,8 @@ from palace.util.datetime_helpers import utc_now from palace.manager.celery.tasks import apply -from palace.manager.integration.license.bibliotheca import BibliothecaAPI -from palace.manager.integration.license.bibliotheca_importer import ( +from palace.manager.integration.license.bibliotheca.api import BibliothecaAPI +from palace.manager.integration.license.bibliotheca.importer import ( DEFAULT_SLICE_SIZE, EVENT_IMPORT_OVERLAP, EVENT_IMPORT_SERVICE_NAME, diff --git a/tests/manager/integration/license/bibliotheca/test_parser.py b/tests/manager/integration/license/bibliotheca/test_parser.py new file mode 100644 index 0000000000..3da427dcac --- /dev/null +++ b/tests/manager/integration/license/bibliotheca/test_parser.py @@ -0,0 +1,592 @@ +from __future__ import annotations + +from datetime import date +from typing import TYPE_CHECKING + +import pytest + +from palace.util.datetime_helpers import datetime_utc + +from palace.manager.api.circulation.data import HoldInfo, LoanInfo +from palace.manager.api.circulation.exceptions import ( + AlreadyCheckedOut, + AlreadyOnHold, + CannotHold, + CirculationException, + CurrentlyAvailable, + NoAvailableCopies, + NoLicenses, + NotCheckedOut, + NotOnHold, + PatronHoldLimitReached, + PatronLoanLimitReached, + RemoteInitiatedServerError, +) +from palace.manager.integration.license.bibliotheca.api import BibliothecaAPI +from palace.manager.integration.license.bibliotheca.parser import ( + BibliothecaParser, + CheckoutResponseParser, + ErrorParser, + EventParser, + ItemListParser, + PatronCirculationParser, +) +from palace.manager.sqlalchemy.model.circulationevent import CirculationEvent +from palace.manager.sqlalchemy.model.classification import Subject +from palace.manager.sqlalchemy.model.contributor import Contributor +from palace.manager.sqlalchemy.model.edition import Edition +from palace.manager.sqlalchemy.model.identifier import Identifier +from palace.manager.sqlalchemy.model.licensing import ( + DeliveryMechanism, + LicensePoolStatus, +) +from palace.manager.sqlalchemy.model.measurement import Measurement +from palace.manager.sqlalchemy.model.resource import Hyperlink, Representation +from tests.manager.integration.license.bibliotheca.conftest import ( + BibliothecaAPITestFixture, +) + +if TYPE_CHECKING: + from tests.fixtures.files import BibliothecaFilesFixture + + +class TestBibliothecaParser: + def test_parse_date(self, bibliotheca_fixture: BibliothecaAPITestFixture): + v = BibliothecaParser.parse_date("2016-01-02T12:34:56") + assert v == datetime_utc(2016, 1, 2, 12, 34, 56) + + assert BibliothecaParser.parse_date(None) is None + assert BibliothecaParser.parse_date("Some weird value") is None + + +class TestEventParser: + def test_parse_empty_list(self, bibliotheca_fixture: BibliothecaAPITestFixture): + data = bibliotheca_fixture.files.sample_data("empty_event_batch.xml") + + # By default, we consider an empty batch of events not + # as an error. + events = list(EventParser().process_all(data)) + assert [] == events + + # But if we consider not having events for a certain time + # period, then an exception should be raised. + no_events_error = True + with pytest.raises(RemoteInitiatedServerError) as excinfo: + list(EventParser().process_all(data, no_events_error)) + assert ( + "No events returned from server. This may not be an error, but treating it as one to be safe." + in str(excinfo.value) + ) + + @pytest.mark.parametrize("data", [b"", b" \n ", ""]) + def test_parse_empty_response_body(self, data: bytes | str): + # Bibliotheca occasionally returns a completely empty response + # body, which cannot be parsed as XML. We treat it the same as a + # response containing no events rather than raising a parse error. + events = list(EventParser().process_all(data)) + assert [] == events + + # And, as with a well-formed empty batch, we raise when the caller + # has indicated that having no events should be treated as an error. + with pytest.raises(RemoteInitiatedServerError) as excinfo: + list(EventParser().process_all(data, no_events_error=True)) + assert ( + "No events returned from server. This may not be an error, but treating it as one to be safe." + in str(excinfo.value) + ) + + def test_parse_empty_end_date_event( + self, bibliotheca_fixture: BibliothecaAPITestFixture + ): + data = bibliotheca_fixture.files.sample_data("empty_end_date_event.xml") + [event] = list(EventParser().process_all(data)) + (threem_id, isbn, patron_id, start_time, end_time, internal_event_type) = event + assert "d5rf89" == threem_id + assert "9781101190623" == isbn + assert None == patron_id + assert datetime_utc(2016, 4, 28, 11, 4, 6) == start_time + assert None == end_time + assert "distributor_license_add" == internal_event_type + + +class TestPatronCirculationParser: + def test_parse(self, bibliotheca_fixture: BibliothecaAPITestFixture): + data = bibliotheca_fixture.files.sample_data("checkouts.xml") + collection = bibliotheca_fixture.collection + loans_and_holds = list(PatronCirculationParser(collection).process_all(data)) + loans = [x for x in loans_and_holds if isinstance(x, LoanInfo)] + holds = [x for x in loans_and_holds if isinstance(x, HoldInfo)] + assert 2 == len(loans) + assert 2 == len(holds) + [l1, l2] = sorted(loans, key=lambda x: str(x.identifier)) + assert "1ad589" == l1.identifier + assert "cgaxr9" == l2.identifier + expect_loan_start = datetime_utc(2015, 3, 20, 18, 50, 22) + expect_loan_end = datetime_utc(2015, 4, 10, 18, 50, 22) + assert expect_loan_start == l1.start_date + assert expect_loan_end == l1.end_date + + [h1, h2] = sorted(holds, key=lambda x: str(x.identifier)) + + # This is the book on reserve. + assert collection.id == h1.collection_id + assert "9wd8" == h1.identifier + expect_hold_start = datetime_utc(2015, 5, 25, 17, 5, 34) + expect_hold_end = datetime_utc(2015, 5, 27, 17, 5, 34) + assert expect_hold_start == h1.start_date + assert expect_hold_end == h1.end_date + assert 0 == h1.hold_position + + # This is the book on hold. + assert "d4o8r9" == h2.identifier + assert collection.id == h2.collection_id + expect_hold_start = datetime_utc(2015, 3, 24, 15, 6, 56) + expect_hold_end = datetime_utc(2015, 3, 24, 15, 7, 51) + assert expect_hold_start == h2.start_date + assert expect_hold_end == h2.end_date + assert 4 == h2.hold_position + + +class TestCheckoutResponseParser: + def test_parse(self, bibliotheca_fixture: BibliothecaAPITestFixture): + data = bibliotheca_fixture.files.sample_data("successful_checkout.xml") + due_date = CheckoutResponseParser().process_first(data) + assert datetime_utc(2015, 4, 16, 0, 32, 36) == due_date + + +class TestErrorParser: + BIBLIOTHECA_ERROR_RESPONSE_BODY_TEMPLATE = ( + '' + "Gen-001" + "{message}" + "" + ) + + @pytest.mark.parametrize( + "incoming_message, error_class, message, debug_message", + [ + ( + "Patron cannot loan more than 12 documents", + PatronLoanLimitReached, + "Patron cannot loan more than 12 documents", + None, + ), + ( + "Patron cannot have more than 15 holds", + PatronHoldLimitReached, + "Patron cannot have more than 15 holds", + None, + ), + ( + "the patron document status was CAN_WISH and not one of CAN_LOAN,RESERVATION", + NoLicenses, + "The library currently has no licenses for this book.", + "the patron document status was CAN_WISH and not one of CAN_LOAN,RESERVATION", + ), + ( + "the patron document status was CAN_HOLD and not one of CAN_LOAN,RESERVATION", + NoAvailableCopies, + "No copies available to check out.", + "the patron document status was CAN_HOLD and not one of CAN_LOAN,RESERVATION", + ), + ( + "the patron document status was LOAN and not one of CAN_LOAN,RESERVATION", + AlreadyCheckedOut, + "You already have this book checked out.", + "the patron document status was LOAN and not one of CAN_LOAN,RESERVATION", + ), + ( + "The patron has no eBooks checked out", + NotCheckedOut, + "The patron has no eBooks checked out", + None, + ), + ( + "the patron document status was CAN_LOAN and not one of CAN_HOLD", + CurrentlyAvailable, + "Cannot place a hold on an available title.", + "the patron document status was CAN_LOAN and not one of CAN_HOLD", + ), + ( + "the patron document status was HOLD and not one of CAN_HOLD", + AlreadyOnHold, + "You already have this book on hold.", + "the patron document status was HOLD and not one of CAN_HOLD", + ), + ( + "The patron does not have the book on hold", + NotOnHold, + "The patron does not have the book on hold", + None, + ), + # This is such a weird case we don't have a special exception for it. + ( + "the patron document status was LOAN and not one of CAN_HOLD", + CannotHold, + "Could not place hold (reason unknown).", + "the patron document status was LOAN and not one of CAN_HOLD", + ), + ], + ) + def test_exception( + self, + incoming_message: str, + error_class: type[CirculationException], + message: str, + debug_message: str | None, + ): + document = self.BIBLIOTHECA_ERROR_RESPONSE_BODY_TEMPLATE.format( + message=incoming_message + ) + error = ErrorParser().process_first(document) + assert error.__class__ is error_class + assert error.problem_detail.detail == message + assert error.problem_detail.debug_message == debug_message + + @pytest.mark.parametrize( + "incoming_message, incoming_message_from_file, error_string", + [ + ( + # Simulate the message we get when the server goes down. + "The server has encountered an error", + None, + "The server has encountered an error", + ), + ( + # Simulate an unexpected response, which is not a unicode string. + b"Beep boop bytes", + None, + "Beep boop bytes", + ), + ( + # Simulate an unexpected response, which cannot be decoded as a string. + b"\xde\xad\xbe\xef", + None, + "Unreadable error message (Unicode decode error).", + ), + ( + # Simulate the message we get when the server gives a vague error. + None, + "error_unknown.xml", + "Unknown error", + ), + ( + # Simulate the message we get when the error message is + # 'Authentication failed' but our authentication information is + # set up correctly. + None, + "error_authentication_failed.xml", + "Authentication failed", + ), + ( + """This error does not follow the standard set out by Bibliotheca.""", + None, + "Unknown error", + ), + ( + # Empty error message + """""", + None, + "Unknown error", + ), + ], + ) + def test_remote_initiated_server_error( + self, + incoming_message: str | bytes | None, + incoming_message_from_file: str | None, + error_string: str, + bibliotheca_files_fixture: BibliothecaFilesFixture, + ): + if incoming_message_from_file: + incoming_message = bibliotheca_files_fixture.sample_text( + incoming_message_from_file + ) + assert incoming_message is not None + error = ErrorParser().process_first(incoming_message) + assert isinstance(error, RemoteInitiatedServerError) + + assert BibliothecaAPI.SERVICE_NAME == error.service_name + assert error_string == str(error) + + problem = error.problem_detail + assert 502 == problem.status_code + assert "Integration error communicating with Bibliotheca" == problem.detail + assert "Third-party service failed." == problem.title + + +class TestBibliothecaEventParser: + # Sample event feed to test out the parser. + TWO_EVENTS = """ + 1b0d6667-a10e-424a-9f73-fb6f6d41308e + 2014-04-14T13:59:05.6920303Z + 2014-04-03T00:00:34 + + + test-library + event-1 + CHECKIN + 2014-04-03T00:00:23 + 2014-04-03T00:00:23 + theitem1 + 900isbn1 + patronid1 + 2014-04-14T13:59:05 + + + test-library + event-2 + CHECKOUT + 2014-04-03T00:00:34 + 2014-04-02T23:57:37 + theitem2 + 900isbn2 + patronid2 + 2014-04-14T13:59:05 + + + +""" + + def test_parse_event_batch(self): + # Parsing the XML gives us two events. + event1, event2 = EventParser().process_all(self.TWO_EVENTS) + + (threem_id, isbn, patron_id, start_time, end_time, internal_event_type) = event1 + + assert "theitem1" == threem_id + assert "900isbn1" == isbn + assert "patronid1" == patron_id + assert CirculationEvent.DISTRIBUTOR_CHECKIN == internal_event_type + assert start_time == end_time + + (threem_id, isbn, patron_id, start_time, end_time, internal_event_type) = event2 + assert "theitem2" == threem_id + assert "900isbn2" == isbn + assert "patronid2" == patron_id + assert CirculationEvent.DISTRIBUTOR_CHECKOUT == internal_event_type + + # Verify that start and end time were parsed correctly. + correct_start = datetime_utc(2014, 4, 3, 0, 0, 34) + correct_end = datetime_utc(2014, 4, 2, 23, 57, 37) + assert correct_start == start_time + assert correct_end == end_time + + +class TestItemListParser: + def test_contributors_for_string(cls): + authors = list( + ItemListParser.contributors_from_string( + "Walsh, Jill Paton; Sayers, Dorothy L." + ) + ) + assert [x.sort_name for x in authors] == [ + "Walsh, Jill Paton", + "Sayers, Dorothy L.", + ] + assert [x.roles for x in authors] == [ + (Contributor.Role.AUTHOR,), + (Contributor.Role.AUTHOR,), + ] + + # Parentheticals are stripped. + [author] = ItemListParser.contributors_from_string( + "Baum, Frank L. (Frank Lyell)" + ) + assert "Baum, Frank L." == author.sort_name + + # Contributors may have two levels of entity reference escaping, + # one of which will have already been handled by the initial parse. + # So, we'll test zero and one escapings here. + authors = list( + ItemListParser.contributors_from_string( + "Raji Codell, Esmé; Raji Codell, Esmé" + ) + ) + author_names = [a.sort_name for a in authors] + assert len(authors) == 2 + assert len(set(author_names)) == 1 + assert all("Raji Codell, Esmé" == name for name in author_names) + + # It's possible to specify some role other than AUTHOR_ROLE. + narrators = list( + ItemListParser.contributors_from_string( + "Callow, Simon; Mann, Bruce; Hagon, Garrick", Contributor.Role.NARRATOR + ) + ) + for narrator in narrators: + assert (Contributor.Role.NARRATOR,) == narrator.roles + assert ["Callow, Simon", "Mann, Bruce", "Hagon, Garrick"] == [ + narrator.sort_name for narrator in narrators + ] + + def test_parse_genre_string(self): + def f(genre_string): + genres = ItemListParser.parse_genre_string(genre_string) + assert all([x.type == Subject.BISAC for x in genres]) + return [x.name for x in genres] + + assert ["Children's Health", "Health"] == f("Children&#39;s Health,Health,") + + assert [ + "Action & Adventure", + "Science Fiction", + "Fantasy", + "Magic", + "Renaissance", + ] == f( + "Action &amp; Adventure,Science Fiction, Fantasy, Magic,Renaissance," + ) + + def test_item_list(self, bibliotheca_fixture: BibliothecaAPITestFixture): + data = bibliotheca_fixture.files.sample_data("item_metadata_list_mini.xml") + data_parsed = list(ItemListParser().process_all(data)) + + # There should be 2 items in the list. + assert 2 == len(data_parsed) + + cooked = data_parsed[0] + + assert "The Incense Game" == cooked.title + assert "A Novel of Feudal Japan" == cooked.subtitle + assert Edition.BOOK_MEDIUM == cooked.medium + assert "eng" == cooked.language + assert "St. Martin's Press" == cooked.publisher + assert date(year=2012, month=9, day=17) == cooked.published + + primary = cooked.primary_identifier_data + assert "ddf4gr9" == primary.identifier + assert Identifier.BIBLIOTHECA_ID == primary.type + + identifiers = sorted(cooked.identifiers, key=lambda x: x.identifier) + assert ["9781250015280", "9781250031112", "ddf4gr9"] == [ + x.identifier for x in identifiers + ] + + [author] = cooked.contributors + assert "Rowland, Laura Joh" == author.sort_name + assert (Contributor.Role.AUTHOR,) == author.roles + + subjects = [x.name for x in cooked.subjects if x.name is not None] + assert ["Children's Health", "Mystery & Detective"] == sorted(subjects) + + [pages] = cooked.measurements + assert Measurement.PAGE_COUNT == pages.quantity_measured + assert 304 == pages.value + + [alternate, image, description] = sorted(cooked.links, key=lambda x: x.rel) + assert "alternate" == alternate.rel + assert alternate.href.startswith("http://ebook.3m.com/library") + + # We have a full-size image... + assert Hyperlink.IMAGE == image.rel + assert Representation.JPEG_MEDIA_TYPE == image.media_type + assert image.href is not None + assert image.href.startswith("http://ebook.3m.com/delivery") + assert "documentID=ddf4gr9" in image.href + assert "&size=NORMAL" not in image.href + + # ... and a thumbnail, which we obtained by adding an argument + # to the main image URL. + thumbnail = image.thumbnail + assert Hyperlink.THUMBNAIL_IMAGE == thumbnail.rel + assert Representation.JPEG_MEDIA_TYPE == thumbnail.media_type + assert thumbnail.href == image.href + "&size=NORMAL" + + # We have a description. + assert Hyperlink.DESCRIPTION == description.rel + assert isinstance(description.content, str) + assert description.content.startswith("Winner") + + def test_multiple_contributor_roles( + self, bibliotheca_fixture: BibliothecaAPITestFixture + ): + data = bibliotheca_fixture.files.sample_data("item_metadata_audio.xml") + [parsed_data] = list(ItemListParser().process_all(data)) + names_and_roles = [] + for c in parsed_data.contributors: + [role] = c.roles + names_and_roles.append((c.sort_name, role)) + + # We found one author and three narrators. + assert sorted( + [ + ("Riggs, Ransom", "Author"), + ("Callow, Simon", "Narrator"), + ("Mann, Bruce", "Narrator"), + ("Hagon, Garrick", "Narrator"), + ] + ) == sorted(names_and_roles) + + def test_circulation_data_status( + self, bibliotheca_fixture: BibliothecaAPITestFixture + ): + """Test that CirculationData from ItemListParser has correct status.""" + data = bibliotheca_fixture.files.sample_data("item_metadata_list_mini.xml") + data_parsed = list(ItemListParser().process_all(data)) + + # Check the first book's circulation data + bibliographic1 = data_parsed[0] + circulation1 = bibliographic1.circulation + + # This book has 1 license, so status should be ACTIVE + assert circulation1.licenses_owned == 1 + assert circulation1.licenses_available == 1 + assert circulation1.status == LicensePoolStatus.ACTIVE + + # Check the second book's circulation data + bibliographic2 = data_parsed[1] + circulation2 = bibliographic2.circulation + + # This book also has licenses, so status should be ACTIVE + assert circulation2.licenses_owned == 1 + assert circulation2.status == LicensePoolStatus.ACTIVE + + def test_circulation_data_status_exhausted( + self, bibliotheca_fixture: BibliothecaAPITestFixture + ): + """Test that CirculationData has EXHAUSTED status when licenses_owned is 0.""" + data = bibliotheca_fixture.files.sample_data("item_metadata_list_mini.xml") + # Replace TotalCopies with 0 to test EXHAUSTED status + data = data.replace( + b"1", b"0" + ) + + data_parsed = list(ItemListParser().process_all(data)) + + # Both books should have EXHAUSTED status + for bibliographic in data_parsed: + circulation = bibliographic.circulation + assert circulation.licenses_owned == 0 + assert circulation.status == LicensePoolStatus.EXHAUSTED + + def test_internal_formats(self): + m = ItemListParser.internal_formats + + def _check_format(input, expect_medium, expect_format, expect_drm): + medium, formats = m(input) + assert medium == expect_medium + [format] = formats + assert expect_format == format.content_type + assert expect_drm == format.drm_scheme + + rep = Representation + adobe = DeliveryMechanism.ADOBE_DRM + findaway = DeliveryMechanism.FINDAWAY_DRM + book = Edition.BOOK_MEDIUM + + # Verify that we handle the known strings from Bibliotheca + # appropriately. + _check_format("EPUB", book, rep.EPUB_MEDIA_TYPE, adobe) + _check_format("EPUB3", book, rep.EPUB_MEDIA_TYPE, adobe) + _check_format("PDF", book, rep.PDF_MEDIA_TYPE, adobe) + _check_format("MP3", Edition.AUDIO_MEDIUM, None, findaway) + + # Now Try a string we don't recognize from Bibliotheca. + medium, formats = m("Unknown") + + # We assume it's a book. + assert Edition.BOOK_MEDIUM == medium + + # But we don't know which format. + assert [] == formats diff --git a/tests/manager/integration/license/test_bibliotheca_purchase_record_importer.py b/tests/manager/integration/license/bibliotheca/test_purchase_record_importer.py similarity index 99% rename from tests/manager/integration/license/test_bibliotheca_purchase_record_importer.py rename to tests/manager/integration/license/bibliotheca/test_purchase_record_importer.py index dc690e8c1c..95a74f5997 100644 --- a/tests/manager/integration/license/test_bibliotheca_purchase_record_importer.py +++ b/tests/manager/integration/license/bibliotheca/test_purchase_record_importer.py @@ -11,8 +11,8 @@ from palace.util.log import LogLevel from palace.manager.celery.tasks import apply -from palace.manager.integration.license.bibliotheca import BibliothecaAPI -from palace.manager.integration.license.bibliotheca_purchase_record_importer import ( +from palace.manager.integration.license.bibliotheca.api import BibliothecaAPI +from palace.manager.integration.license.bibliotheca.purchase_record_importer import ( _MARC_PAGE_SIZE, DEFAULT_PURCHASE_RECORD_START_TIME, PURCHASE_RECORD_SERVICE_NAME, diff --git a/tests/manager/integration/license/test_bibliotheca_scripts.py b/tests/manager/integration/license/bibliotheca/test_scripts.py similarity index 98% rename from tests/manager/integration/license/test_bibliotheca_scripts.py rename to tests/manager/integration/license/bibliotheca/test_scripts.py index b0755c3153..56ef732d7c 100644 --- a/tests/manager/integration/license/test_bibliotheca_scripts.py +++ b/tests/manager/integration/license/bibliotheca/test_scripts.py @@ -9,10 +9,10 @@ from palace.util.exceptions import PalaceValueError from palace.manager.celery.tasks import bibliotheca -from palace.manager.integration.license.bibliotheca_purchase_record_importer import ( +from palace.manager.integration.license.bibliotheca.purchase_record_importer import ( DEFAULT_PURCHASE_RECORD_START_TIME, ) -from palace.manager.integration.license.bibliotheca_scripts import ( +from palace.manager.integration.license.bibliotheca.scripts import ( CirculationUpdateCollection, ImportEventCollection, ImportPurchaseRecordCollection, diff --git a/tests/manager/scripts/test_configuration.py b/tests/manager/scripts/test_configuration.py index 3d2cf1ad1c..3d1198a81e 100644 --- a/tests/manager/scripts/test_configuration.py +++ b/tests/manager/scripts/test_configuration.py @@ -4,7 +4,7 @@ import pytest -from palace.manager.integration.license.bibliotheca import BibliothecaAPI +from palace.manager.integration.license.bibliotheca.api import BibliothecaAPI from palace.manager.integration.license.overdrive.api import OverdriveAPI from palace.manager.scripts.configuration import ( ConfigureCollectionScript, diff --git a/tests/manager/scripts/test_informational.py b/tests/manager/scripts/test_informational.py index 2343af1406..81bc0e4cc2 100644 --- a/tests/manager/scripts/test_informational.py +++ b/tests/manager/scripts/test_informational.py @@ -4,7 +4,7 @@ from unittest.mock import call, create_autospec from palace.manager.integration.goals import Goals -from palace.manager.integration.license.bibliotheca import BibliothecaAPI +from palace.manager.integration.license.bibliotheca.api import BibliothecaAPI from palace.manager.integration.license.overdrive.api import OverdriveAPI from palace.manager.scripts.informational import ( Explain, diff --git a/tests/manager/scripts/test_monitor.py b/tests/manager/scripts/test_monitor.py index 9fc2ab96e9..67c5787c6b 100644 --- a/tests/manager/scripts/test_monitor.py +++ b/tests/manager/scripts/test_monitor.py @@ -3,7 +3,7 @@ import pytest from palace.manager.core.monitor import CollectionMonitor, Monitor -from palace.manager.integration.license.bibliotheca import BibliothecaAPI +from palace.manager.integration.license.bibliotheca.api import BibliothecaAPI from palace.manager.integration.license.opds.opds1.api import OPDSAPI from palace.manager.scripts.monitor import ( RunCollectionMonitorScript, diff --git a/tests/manager/sqlalchemy/model/test_collection.py b/tests/manager/sqlalchemy/model/test_collection.py index 03ac15b962..028fe98285 100644 --- a/tests/manager/sqlalchemy/model/test_collection.py +++ b/tests/manager/sqlalchemy/model/test_collection.py @@ -9,7 +9,7 @@ from palace.manager.integration.base import integration_settings_update from palace.manager.integration.goals import Goals -from palace.manager.integration.license.bibliotheca import BibliothecaAPI +from palace.manager.integration.license.bibliotheca.api import BibliothecaAPI from palace.manager.integration.license.boundless.api import BoundlessApi from palace.manager.integration.license.opds.opds1.settings import OPDSImporterSettings from palace.manager.integration.license.overdrive.api import OverdriveAPI diff --git a/tests/mocks/bibliotheca.py b/tests/mocks/bibliotheca.py index 5f8a026b1f..23fd8435b8 100644 --- a/tests/mocks/bibliotheca.py +++ b/tests/mocks/bibliotheca.py @@ -2,7 +2,7 @@ from sqlalchemy.orm import Session -from palace.manager.integration.license.bibliotheca import BibliothecaAPI +from palace.manager.integration.license.bibliotheca.api import BibliothecaAPI from palace.manager.sqlalchemy.model.collection import Collection from palace.manager.sqlalchemy.model.library import Library from palace.manager.util.http.base import raise_for_bad_response