diff --git a/alembic/versions/20260624_5ca948100902_drop_coveragerecords_and_.py b/alembic/versions/20260624_5ca948100902_drop_coveragerecords_and_.py new file mode 100644 index 0000000000..667fddcb4e --- /dev/null +++ b/alembic/versions/20260624_5ca948100902_drop_coveragerecords_and_.py @@ -0,0 +1,197 @@ +"""Drop coveragerecords and equivalentscoveragerecords tables + +The CoverageProvider machinery that read and wrote ``coveragerecords`` was +retired in the previous release, and the equivalent-identifiers refresh that +wrote ``equivalentscoveragerecords`` moved to a Celery task before that. No +running code (current or N-1) references either table, so both are dropped here +along with their shared ``coverage_status`` enum type. + +The ``timestamps`` table and its separate ``service_type`` enum are unaffected. + +Revision ID: 5ca948100902 +Revises: a6c85605404c +Create Date: 2026-06-24 22:47:03.130552+00:00 + +""" + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "5ca948100902" +down_revision = "a6c85605404c" +branch_labels = None +depends_on = None + +# The status enum shared by the two dropped tables. We manage the Postgres type +# explicitly (create_type=False) so it is created/dropped exactly once rather +# than once per table. +coverage_status = postgresql.ENUM( + "success", + "transient failure", + "persistent failure", + "registered", + name="coverage_status", + create_type=False, +) + + +def upgrade() -> None: + op.drop_index( + op.f("ix_equivalentscoveragerecords_equivalency_id"), + table_name="equivalentscoveragerecords", + ) + op.drop_index( + op.f("ix_equivalentscoveragerecords_operation"), + table_name="equivalentscoveragerecords", + ) + op.drop_index( + op.f("ix_equivalentscoveragerecords_status"), + table_name="equivalentscoveragerecords", + ) + op.drop_index( + op.f("ix_equivalentscoveragerecords_timestamp"), + table_name="equivalentscoveragerecords", + ) + op.drop_table("equivalentscoveragerecords") + + op.drop_index( + op.f("ix_coveragerecords_data_source_id_operation_identifier_id"), + table_name="coveragerecords", + ) + op.drop_index(op.f("ix_coveragerecords_exception"), table_name="coveragerecords") + op.drop_index( + op.f("ix_coveragerecords_identifier_id"), table_name="coveragerecords" + ) + op.drop_index(op.f("ix_coveragerecords_status"), table_name="coveragerecords") + op.drop_index(op.f("ix_coveragerecords_timestamp"), table_name="coveragerecords") + op.drop_index( + "ix_identifier_id_data_source_id_operation", table_name="coveragerecords" + ) + op.drop_index( + "ix_identifier_id_data_source_id_operation_collection_id", + table_name="coveragerecords", + ) + op.drop_table("coveragerecords") + + # The coverage_status enum was used only by the two tables just dropped. + coverage_status.drop(op.get_bind(), checkfirst=False) + + +def downgrade() -> None: + coverage_status.create(op.get_bind(), checkfirst=False) + + op.create_table( + "coveragerecords", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("identifier_id", sa.Integer(), nullable=True), + sa.Column("data_source_id", sa.Integer(), nullable=True), + sa.Column("operation", sa.String(length=255), nullable=True), + sa.Column("timestamp", sa.DateTime(timezone=True), nullable=True), + sa.Column("status", coverage_status, nullable=True), + sa.Column("exception", sa.Unicode(), nullable=True), + sa.Column("collection_id", sa.Integer(), nullable=True), + sa.ForeignKeyConstraint( + ["collection_id"], + ["collections.id"], + name=op.f("coveragerecords_collection_id_fkey"), + ), + sa.ForeignKeyConstraint( + ["data_source_id"], + ["datasources.id"], + name=op.f("coveragerecords_data_source_id_fkey"), + ), + sa.ForeignKeyConstraint( + ["identifier_id"], + ["identifiers.id"], + name=op.f("coveragerecords_identifier_id_fkey"), + ), + sa.PrimaryKeyConstraint("id", name=op.f("coveragerecords_pkey")), + ) + op.create_index( + "ix_identifier_id_data_source_id_operation_collection_id", + "coveragerecords", + ["identifier_id", "data_source_id", "operation", "collection_id"], + unique=True, + ) + op.create_index( + "ix_identifier_id_data_source_id_operation", + "coveragerecords", + ["identifier_id", "data_source_id", "operation"], + unique=True, + postgresql_where=sa.text("collection_id IS NULL"), + ) + op.create_index( + op.f("ix_coveragerecords_timestamp"), + "coveragerecords", + ["timestamp"], + unique=False, + ) + op.create_index( + op.f("ix_coveragerecords_status"), "coveragerecords", ["status"], unique=False + ) + op.create_index( + op.f("ix_coveragerecords_identifier_id"), + "coveragerecords", + ["identifier_id"], + unique=False, + ) + op.create_index( + op.f("ix_coveragerecords_exception"), + "coveragerecords", + ["exception"], + unique=False, + ) + op.create_index( + op.f("ix_coveragerecords_data_source_id_operation_identifier_id"), + "coveragerecords", + ["data_source_id", "operation", "identifier_id"], + unique=False, + ) + + op.create_table( + "equivalentscoveragerecords", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("equivalency_id", sa.Integer(), nullable=False), + sa.Column("operation", sa.String(length=255), nullable=True), + sa.Column("timestamp", sa.DateTime(timezone=True), nullable=True), + sa.Column("status", coverage_status, nullable=True), + sa.Column("exception", sa.Unicode(), nullable=True), + sa.ForeignKeyConstraint( + ["equivalency_id"], + ["equivalents.id"], + name=op.f("equivalentscoveragerecords_equivalency_id_fkey"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("equivalentscoveragerecords_pkey")), + sa.UniqueConstraint( + "equivalency_id", + "operation", + name=op.f("equivalentscoveragerecords_equivalency_id_operation_key"), + ), + ) + op.create_index( + op.f("ix_equivalentscoveragerecords_timestamp"), + "equivalentscoveragerecords", + ["timestamp"], + unique=False, + ) + op.create_index( + op.f("ix_equivalentscoveragerecords_status"), + "equivalentscoveragerecords", + ["status"], + unique=False, + ) + op.create_index( + op.f("ix_equivalentscoveragerecords_operation"), + "equivalentscoveragerecords", + ["operation"], + unique=False, + ) + op.create_index( + op.f("ix_equivalentscoveragerecords_equivalency_id"), + "equivalentscoveragerecords", + ["equivalency_id"], + unique=False, + ) diff --git a/src/palace/manager/api/admin/controller/work_editor.py b/src/palace/manager/api/admin/controller/work_editor.py index 194e41798a..970cac0bab 100644 --- a/src/palace/manager/api/admin/controller/work_editor.py +++ b/src/palace/manager/api/admin/controller/work_editor.py @@ -20,7 +20,6 @@ INVALID_RATING, INVALID_SERIES_POSITION, METADATA_REFRESH_FAILURE, - METADATA_REFRESH_PENDING, MISSING_CUSTOM_LIST, UNKNOWN_LANGUAGE, UNKNOWN_MEDIUM, @@ -31,7 +30,6 @@ ) from palace.manager.api.problem_details import ( LIBRARY_NOT_FOUND, - REMOTE_INTEGRATION_FAILED, ) from palace.manager.api.util.flask import get_request_library from palace.manager.core.classifier import NO_NUMBER, NO_VALUE, genres @@ -448,9 +446,15 @@ def unsuppress( ) def refresh_metadata( - self, identifier_type: str, identifier: str, provider: Any | None = None + self, identifier_type: str, identifier: str ) -> Response | ProblemDetail: - """Refresh the metadata for a book from the content server""" + """Refresh the metadata for a book from the content server. + + Metadata refresh used to run through the per-source CoverageProvider + machinery, which has been retired. No provider is wired up, so this + endpoint is retained for API compatibility but no longer performs a + refresh; it always reports failure. + """ library = get_request_library() self.require_librarian(library) @@ -458,29 +462,7 @@ def refresh_metadata( if isinstance(work, ProblemDetail): return work - if provider is None: - return METADATA_REFRESH_FAILURE - - assert work.presentation_edition is not None - primary_identifier = work.presentation_edition.primary_identifier - try: - record = provider.ensure_coverage(primary_identifier, force=True) - except Exception: - # The coverage provider may raise an HTTPIntegrationException. - return REMOTE_INTEGRATION_FAILED - - if record.exception: - # There was a coverage failure. - if str(record.exception).startswith("201") or str( - record.exception - ).startswith("202"): - # A 201/202 error means it's never looked up this work before - # so it's started the resolution process or looking for sources. - return METADATA_REFRESH_PENDING - # Otherwise, it just doesn't know anything. - return METADATA_REFRESH_FAILURE - - return Response("", 200) + return METADATA_REFRESH_FAILURE def classifications( self, identifier_type: str, identifier: str diff --git a/src/palace/manager/celery/tasks/apply.py b/src/palace/manager/celery/tasks/apply.py index 5dd4ab93c1..3fa853a1be 100644 --- a/src/palace/manager/celery/tasks/apply.py +++ b/src/palace/manager/celery/tasks/apply.py @@ -107,9 +107,7 @@ def bibliographic_apply( load_from_id(session, Collection, collection_id) if collection_id else None ) - bibliographic.apply( - session, edition, collection, replace, create_coverage_record=False - ) + bibliographic.apply(session, edition, collection, replace) class ApplyBibliographicCallable(Protocol): diff --git a/src/palace/manager/core/coverage.py b/src/palace/manager/core/coverage.py deleted file mode 100644 index 033858d111..0000000000 --- a/src/palace/manager/core/coverage.py +++ /dev/null @@ -1,1364 +0,0 @@ -import logging -import traceback - -from sqlalchemy.orm.session import Session -from sqlalchemy.sql.functions import func - -from palace.util.datetime_helpers import utc_now - -from palace.manager.core.monitor import TimestampData -from palace.manager.data_layer.policy.replacement import ReplacementPolicy -from palace.manager.service.container import container_instance -from palace.manager.sqlalchemy.model.collection import Collection, CollectionMissing -from palace.manager.sqlalchemy.model.coverage import ( - BaseCoverageRecord, - CoverageRecord, - Timestamp, -) -from palace.manager.sqlalchemy.model.datasource import DataSource -from palace.manager.sqlalchemy.model.edition import Edition -from palace.manager.sqlalchemy.model.identifier import Identifier -from palace.manager.sqlalchemy.model.licensing import LicensePool -from palace.manager.sqlalchemy.util import get_one - - -class CoverageFailure: - """Object representing the failure to provide coverage.""" - - def __init__( - self, obj, exception, data_source=None, transient=True, collection=None - ): - self.obj = obj - self.data_source = data_source - self.exception = exception - self.transient = transient - self.collection = collection - - def __repr__(self): - if self.data_source: - data_source = self.data_source.name - else: - data_source = None - return "".format( - self.obj, - data_source, - self.transient, - self.exception, - ) - - def to_coverage_record(self, operation=None): - """Convert this failure into a CoverageRecord.""" - if not self.data_source: - raise Exception( - "Cannot convert coverage failure to CoverageRecord because it has no output source." - ) - - record, ignore = CoverageRecord.add_for( - self.obj, self.data_source, operation=operation, collection=self.collection - ) - record.exception = self.exception - if self.transient: - record.status = CoverageRecord.TRANSIENT_FAILURE - else: - record.status = CoverageRecord.PERSISTENT_FAILURE - return record - - -class CoverageProviderProgress(TimestampData): - """A TimestampData optimized for the special needs of - CoverageProviders. - """ - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - # The offset is distinct from the counter, in that it's not written - # to the database -- it's used to track state that's necessary within - # a single run of the CoverageProvider. - self.offset = 0 - - self.successes = 0 - self.transient_failures = 0 - self.persistent_failures = 0 - - @property - def achievements(self): - """Represent the achievements of a CoverageProvider as a - human-readable string. - """ - template = "Items processed: %d. Successes: %d, transient failures: %d, persistent failures: %d" - total = self.successes + self.transient_failures + self.persistent_failures - return template % ( - total, - self.successes, - self.transient_failures, - self.persistent_failures, - ) - - @achievements.setter - def achievements(self, value): - # It's not possible to set .achievements directly. Do nothing. - pass - - -class BaseCoverageProvider: - """Run certain objects through an algorithm. If the algorithm returns - success, add a coverage record for that object, so the object - doesn't need to be processed again. If the algorithm returns a - CoverageFailure, that failure may itself be memorialized as a - coverage record. - - Instead of instantiating this class directly, subclass one of its - subclasses, e.g. IdentifierCoverageProvider. - - In IdentifierCoverageProvider the 'objects' are Identifier objects - and the coverage records are CoverageRecord objects. - """ - - # In your subclass, set this to the name of the service, - # e.g. "Overdrive Bibliographic Coverage Provider". - SERVICE_NAME: str | None = None - - # In your subclass, you _may_ set this to a string that distinguishes - # two different CoverageProviders from the same data source. - # (You may also override the operation method, if you need - # database access to determine which operation to use.) - OPERATION: str | None = None - - # The database session will be committed each time the - # BaseCoverageProvider has (attempted to) provide coverage to this - # number of Identifiers. You may change this in your subclass. - # It's also possible to change it by passing in a value for - # `batch_size` in the constructor, but generally nobody bothers - # doing this. - DEFAULT_BATCH_SIZE: int = 100 - - def __init__( - self, - _db, - batch_size=None, - cutoff_time=None, - registered_only=False, - ): - """Constructor. - - :param batch_size: The maximum number of objects that will be processed - at once. - - :param cutoff_time: Coverage records created before this time - will be treated as though they did not exist. - - :param registered_only: Optional. Determines whether this - CoverageProvider will only cover items that already have been - "preregistered" with a CoverageRecord with a registered or failing - status. This option is only used on the Metadata Wrangler. - """ - self._db = _db - if not self.__class__.SERVICE_NAME: - raise ValueError("%s must define SERVICE_NAME." % self.__class__.__name__) - service_name = self.__class__.SERVICE_NAME - operation = self.operation - if operation: - service_name += " (%s)" % operation - self.service_name = service_name - if not batch_size or batch_size < 0: - batch_size = self.DEFAULT_BATCH_SIZE - self.batch_size = batch_size - self.cutoff_time = cutoff_time - self.registered_only = registered_only - self.collection_id = None - - # Call init_resources() to initialize the logging configuration. - self.services = container_instance() - self.services.init_resources() - - @property - def log(self): - if not hasattr(self, "_log"): - self._log = logging.getLogger(self.service_name) - return self._log - - @property - def collection(self): - """Retrieve the Collection object associated with this - CoverageProvider. - """ - if not self.collection_id: - return None - return get_one(self._db, Collection, id=self.collection_id) - - @property - def operation(self): - """Which operation should this CoverageProvider use to - distinguish between multiple CoverageRecords from the same data - source? - """ - return self.OPERATION - - def run(self): - start = utc_now() - result = self.run_once_and_update_timestamp() - result = result or CoverageProviderProgress() - self.finalize_timestampdata(result, start=start) - return result - - def run_once_and_update_timestamp(self): - # First prioritize items that have never had a coverage attempt before. - # Then cover items that failed with a transient failure on a - # previous attempt. - covered_status_lists = [ - BaseCoverageRecord.PREVIOUSLY_ATTEMPTED, - BaseCoverageRecord.DEFAULT_COUNT_AS_COVERED, - ] - start_time = utc_now() - timestamp = self.timestamp - - # We'll use this TimestampData object to track our progress - # as we grant coverage to items. - progress = CoverageProviderProgress(start=start_time) - - for covered_statuses in covered_status_lists: - # We may have completed our work for the previous value of - # covered_statuses, but there's more work to do. Unset the - # 'finish' date to guarantee that progress.is_complete - # starts out False. - # - # Also set the offset to zero to ensure that we always start - # at the start of the database table. - original_finish = progress.finish = None - progress.offset = 0 - - # Call run_once() until we get an exception or - # progress.finish is set. - while not progress.is_complete: - try: - new_progress = self.run_once( - progress, count_as_covered=covered_statuses - ) - # run_once can either return a new - # CoverageProviderProgress object, or modify - # in-place the one it was passed. - if new_progress is not None: - progress = new_progress - except Exception as e: - logging.error( - "CoverageProvider %s raised uncaught exception.", - self.service_name, - exc_info=e, - ) - progress.exception = traceback.format_exc() - progress.finish = utc_now() - - # The next run_once() call might raise an exception, - # so let's write the work to the database as it's - # done. - original_finish = progress.finish - self.finalize_timestampdata(progress) - - # That wrote a value for progress.finish to the - # database, which is fine, but we don't necessarily - # want that value for progress.finish to stand. It - # might incorrectly make progress.is_complete appear - # to be True, making us exit the loop before we mean - # to. - if not progress.exception: - progress.finish = original_finish - - # TODO: We should be able to return a list of progress objects, - # not just one. - return progress - - @property - def timestamp(self): - """Look up the Timestamp object for this CoverageProvider.""" - return Timestamp.lookup( - self._db, - self.service_name, - Timestamp.COVERAGE_PROVIDER_TYPE, - collection=self.collection, - ) - - def finalize_timestampdata(self, timestamp, **kwargs): - """Finalize the given TimestampData and write it to the - database. - """ - timestamp.finalize( - self.service_name, - Timestamp.COVERAGE_PROVIDER_TYPE, - collection=self.collection, - **kwargs, - ) - timestamp.apply(self._db) - self._db.commit() - - def run_once(self, progress, count_as_covered=None): - """Try to grant coverage to a number of uncovered items. - - NOTE: If you override this method, it's very important that - your implementation eventually do one of the following: - * Set progress.finish - * Set progress.exception - * Raise an exception - - If you don't do any of these things, run() will assume you still - have work to do, and will keep calling run_once() forever. - - :param progress: A CoverageProviderProgress representing the - progress made so far, and the number of records that - need to be ignored for the rest of the run. - - :param count_as_covered: Which values for CoverageRecord.status - should count as meaning 'already covered'. - - :return: A CoverageProviderProgress representing whatever - additional progress has been made. - """ - count_as_covered = ( - count_as_covered or BaseCoverageRecord.DEFAULT_COUNT_AS_COVERED - ) - # Make it clear which class of items we're covering on this - # run. - count_as_covered_message = " (counting %s as covered)" % ( - ", ".join(count_as_covered) - ) - - qu = self.items_that_need_coverage(count_as_covered=count_as_covered) - - # Running this statement only ONCE as it adds an unbounded query, very bad for the DB cluster - if progress.successes == 0: - self.log.info( - "%d items need coverage%s", qu.count(), count_as_covered_message - ) - else: - self.log.info( - "Covering items after %d successes: %s", - progress.successes, - count_as_covered_message, - ) - - qu = qu.offset(progress.offset) - batch = qu.limit(self.batch_size) - batch_results = batch.all() - batch_count = len(batch_results) - - if not batch_count: - # The batch is empty. We're done. - progress.finish = utc_now() - return progress - - ( - successes, - transient_failures, - persistent_failures, - ), results = self.process_batch_and_handle_results(batch_results) - self.finalize_batch() - # Update the running totals so that the service's eventual timestamp - # will have a useful .achievements. - progress.successes += successes - progress.transient_failures += transient_failures - progress.persistent_failures += persistent_failures - - if BaseCoverageRecord.SUCCESS not in count_as_covered: - # If any successes happened in this batch, increase the - # offset to ignore them, or they will just show up again - # the next time we run this batch. - progress.offset += successes - - if BaseCoverageRecord.TRANSIENT_FAILURE not in count_as_covered: - # If any transient failures happened in this batch, - # increase the offset to ignore them, or they will - # just show up again the next time we run this batch. - progress.offset += transient_failures - - if BaseCoverageRecord.PERSISTENT_FAILURE not in count_as_covered: - # If any persistent failures happened in this batch, - # increase the offset to ignore them, or they will - # just show up again the next time we run this batch. - progress.offset += persistent_failures - - return progress - - def process_batch_and_handle_results(self, batch): - """:return: A 2-tuple (counts, records). - - `counts` is a 3-tuple (successes, transient failures, - persistent_failures). - - `records` is a mixed list of coverage record objects (for - successes and persistent failures) and CoverageFailure objects - (for transient failures). - """ - - # Batch is a query that may not be ordered, so it may return - # different results when executed multiple times. Converting to - # a list ensures that all subsequent code will run on the same items. - batch = list(batch) - - offset_increment = 0 - results = self.process_batch(batch) - successes = 0 - transient_failures = 0 - persistent_failures = 0 - num_ignored = 0 - records = [] - - unhandled_items = set(batch) - success_items = [] - for item in results: - if isinstance(item, CoverageFailure): - if item.obj in unhandled_items: - unhandled_items.remove(item.obj) - record = self.record_failure_as_coverage_record(item) - if item.transient: - self.log.warning( - "Transient failure covering %r: %s", item.obj, item.exception - ) - record.status = BaseCoverageRecord.TRANSIENT_FAILURE - transient_failures += 1 - else: - self.log.error( - "Persistent failure covering %r: %s", item.obj, item.exception - ) - record.status = BaseCoverageRecord.PERSISTENT_FAILURE - persistent_failures += 1 - records.append(record) - else: - # Count this as a success and prepare to add a - # coverage record for it. It won't show up anymore, on - # this run or subsequent runs. - if item in unhandled_items: - unhandled_items.remove(item) - successes += 1 - success_items.append(item) - - records.extend(self.add_coverage_records_for(success_items)) - - # Perhaps some records were ignored--they neither succeeded nor - # failed. Treat them as transient failures. - for item in unhandled_items: - self.log.warning( - "%r was ignored by a coverage provider that was supposed to cover it.", - item, - ) - failure = self.failure_for_ignored_item(item) - record = self.record_failure_as_coverage_record(failure) - record.status = BaseCoverageRecord.TRANSIENT_FAILURE - records.append(record) - num_ignored += 1 - - self.log.info( - "Batch processed with %d successes, %d transient failures, %d persistent failures, %d ignored.", - successes, - transient_failures, - persistent_failures, - num_ignored, - ) - - # For all purposes outside this method, treat an ignored identifier - # as a transient failure. - transient_failures += num_ignored - - return (successes, transient_failures, persistent_failures), records - - def process_batch(self, batch): - """Do what it takes to give coverage records to a batch of - items. - - :return: A mixed list of coverage records and CoverageFailures. - """ - results = [] - for item in batch: - result = self.process_item(item) - if not isinstance(result, CoverageFailure): - self.handle_success(item) - results.append(result) - return results - - def add_coverage_records_for(self, items): - """Add CoverageRecords for a group of items from a batch, - each of which was successful. - """ - return [self.add_coverage_record_for(item) for item in items] - - def handle_success(self, item): - """Do something special to mark the successful coverage of the - given item. - """ - - def should_update(self, coverage_record): - """Should we do the work to update the given coverage record?""" - if coverage_record is None: - # An easy decision -- there is no existing coverage record, - # so we need to do the work. - return True - - if coverage_record.status == BaseCoverageRecord.REGISTERED: - # There's a CoverageRecord, but coverage hasn't actually - # been attempted. Try to get covered. - return True - - if self.cutoff_time is None: - # An easy decision -- without a cutoff_time, once we - # create a coverage record we never update it. - return False - - # We update a coverage record if it was last updated before - # cutoff_time. - return coverage_record.timestamp < self.cutoff_time - - def finalize_batch(self): - """Do whatever is necessary to complete this batch before moving on to - the next one. - - e.g. committing the database session or uploading a bunch of - assets to S3. - """ - self._db.commit() - - # - # Subclasses must implement these virtual methods. - # - - def items_that_need_coverage(self, identifiers=None, **kwargs): - """Create a database query returning only those items that - need coverage. - - :param subset: A list of Identifier objects. If present, return - only items that need coverage *and* are associated with one - of these identifiers. - - Implemented in CoverageProvider. - """ - raise NotImplementedError() - - def add_coverage_record_for(self, item): - """Add a coverage record for the given item. - - Implemented in IdentifierCoverageProvider - """ - raise NotImplementedError() - - def record_failure_as_coverage_record(self, failure): - """Convert the given CoverageFailure to a coverage record. - - Implemented in IdentifierCoverageProvider - """ - raise NotImplementedError() - - def failure_for_ignored_item(self, work): - """Create a CoverageFailure recording the coverage provider's - failure to even try to process an item. - - Implemented in IdentifierCoverageProvider. - """ - raise NotImplementedError() - - def process_item(self, item): - """Do the work necessary to give coverage to one specific item. - - Since this is where the actual work happens, this is not - implemented in IdentifierCoverageProvider, and must be handled in a subclass. - """ - raise NotImplementedError() - - -class IdentifierCoverageProvider(BaseCoverageProvider): - """Run Identifiers of certain types (ISBN, Overdrive, OCLC Number, - etc.) through an algorithm associated with a certain DataSource. - - This class is designed to be subclassed rather than instantiated - directly. Subclasses should define SERVICE_NAME, OPERATION - (optional), DATA_SOURCE_NAME, and - INPUT_IDENTIFIER_TYPES. SERVICE_NAME and OPERATION are described - in BaseCoverageProvider; the rest are described in appropriate - comments in this class. - """ - - # In your subclass, set this to the name of the data source you - # consult when providing coverage, e.g. DataSource.OVERDRIVE. - DATA_SOURCE_NAME: str - - # In your subclass, set this to a single identifier type, or a list - # of identifier types. The CoverageProvider will attempt to give - # coverage to every Identifier of this type. - # - # Setting this to None will attempt to give coverage to every single - # Identifier in the system, which is probably not what you want. - NO_SPECIFIED_TYPES = object() - INPUT_IDENTIFIER_TYPES: None | str | object = NO_SPECIFIED_TYPES - - # Set this to False if a given Identifier needs to be run through - # this CoverageProvider once for every Collection that has this - # Identifier in its catalog. If this is set to True, a given - # Identifier will be considered completely covered the first time - # it's run through this CoverageProvider, no matter how many - # Collections the Identifier belongs to. - COVERAGE_COUNTS_FOR_EVERY_COLLECTION = True - - def __init__( - self, - _db, - collection=None, - input_identifiers=None, - replacement_policy=None, - **kwargs, - ): - """Constructor. - - :param collection: Optional. If information comes in from a - third party about a license pool associated with an - Identifier, the LicensePool that belongs to this Collection - will be used to contain that data. You may pass in None for - this value, but that means that no circulation information - (such as the formats in which a book is available) will be - stored as a result of running this CoverageProvider. Only - bibliographic information will be stored. - :param input_identifiers: Optional. This CoverageProvider is - requested to provide coverage for these specific - Identifiers. - :param replacement_policy: Optional. A ReplacementPolicy to use - when updating local data with data from the third party. - """ - super().__init__(_db, **kwargs) - - # We store the collection ID rather than the Collection to - # avoid breakage if an app server with a scoped session ever - # uses a IdentifierCoverageProvider. - self.collection_id = None - if collection: - self.collection_id = collection.id - self.input_identifiers = input_identifiers - self.replacement_policy = ( - replacement_policy or self._default_replacement_policy(_db) - ) - - if not self.DATA_SOURCE_NAME: - raise ValueError( - "%s must define DATA_SOURCE_NAME" % self.__class__.__name__ - ) - - # Get this information immediately so that an error happens immediately - # if INPUT_IDENTIFIER_TYPES is not set properly. - self.input_identifier_types = self._input_identifier_types() - - def _default_replacement_policy(self, _db): - """Unless told otherwise, assume that we are getting - this data from a reliable metadata source. - """ - return ReplacementPolicy.from_metadata_source() - - @property - def collection_or_not(self): - """If this CoverageProvider needs to be run multiple times on - the same identifier in different collections, this - returns the collection. Otherwise, this returns None. - """ - if self.COVERAGE_COUNTS_FOR_EVERY_COLLECTION: - return None - return self.collection - - @classmethod - def _input_identifier_types(cls): - """Create a normalized value for `input_identifier_types` - based on the INPUT_IDENTIFIER_TYPES class variable. - """ - value = cls.INPUT_IDENTIFIER_TYPES - - # Nip in the bud a situation where someone subclassed this - # class without thinking about a value for - # INPUT_IDENTIFIER_TYPES. - if value is cls.NO_SPECIFIED_TYPES: - raise ValueError( - "%s must define INPUT_IDENTIFIER_TYPES, even if the value is None." - % (cls.__name__) - ) - - if not value: - # We will be processing every single type of identifier in - # the system. This (hopefully) means that the identifiers - # are restricted in some other way, such as being licensed - # to a specific Collection. - return None - elif not isinstance(value, list): - # We will be processing every identifier of a given type. - return [value] - else: - # We will be processing every identify whose type belongs to - # a list of types. - return value - - @classmethod - def register( - cls, - identifier, - data_source=None, - collection=None, - force=False, - autocreate=False, - ): - """Registers an identifier for future coverage. - - See `CoverageProvider.bulk_register` for more information about using - this method. - """ - name = cls.SERVICE_NAME or cls.__name__ - log = logging.getLogger(name) - - new_records, ignored_identifiers = cls.bulk_register( - [identifier], - data_source=data_source, - collection=collection, - force=force, - autocreate=autocreate, - ) - was_registered = identifier not in ignored_identifiers - - new_record = None - if new_records: - [new_record] = new_records - - if was_registered and new_record: - log.info("CREATED %r" % new_record) - return new_record, was_registered - - _db = Session.object_session(identifier) - data_source = cls._data_source_for_registration( - _db, data_source, autocreate=autocreate - ) - - if collection and cls.COVERAGE_COUNTS_FOR_EVERY_COLLECTION: - # There's no need for a collection when registering this - # Identifier, even if it provided the DataSource. - collection = None - - existing_record = CoverageRecord.lookup( - identifier, data_source, cls.OPERATION, collection=collection - ) - log.info("FOUND %r" % existing_record) - return existing_record, was_registered - - @classmethod - def bulk_register( - cls, - identifiers, - data_source=None, - collection=None, - force=False, - autocreate=False, - ): - """Registers identifiers for future coverage. - - This method is primarily for use with CoverageProviders that use the - `registered_only` flag to process items. It's currently only in use - on the Metadata Wrangler. - - :param data_source: DataSource object or basestring representing a - DataSource name. - :param collection: Collection object to be associated with the - CoverageRecords. - :param force: When True, even existing CoverageRecords will have - their status reset to CoverageRecord.REGISTERED. - :param autocreate: When True, a basestring provided by data_source will - be autocreated in the database if it didn't previously exist. - - :return: A tuple of two lists: the first has fresh new REGISTERED - CoverageRecords and the second list already has Identifiers that - were ignored because they already had coverage. - - TODO: Take identifier eligibility into account when registering. - """ - if not identifiers: - return list(), list() - - _db = Session.object_session(identifiers[0]) - data_source = cls._data_source_for_registration( - _db, data_source, autocreate=autocreate - ) - - if collection and cls.COVERAGE_COUNTS_FOR_EVERY_COLLECTION: - # There's no need for a collection on this CoverageRecord. - collection = None - - new_records, ignored_identifiers = CoverageRecord.bulk_add( - identifiers, - data_source, - operation=cls.OPERATION, - status=CoverageRecord.REGISTERED, - collection=collection, - force=force, - ) - - return new_records, ignored_identifiers - - @classmethod - def _data_source_for_registration(cls, _db, data_source, autocreate=False): - """Finds or creates a DataSource for the registration methods - `cls.register` and `cls.bulk_register`. - """ - if not data_source: - return DataSource.lookup(_db, cls.DATA_SOURCE_NAME) - if isinstance(data_source, DataSource): - return data_source - if isinstance(data_source, (bytes, str)): - return DataSource.lookup(_db, data_source, autocreate=autocreate) - - @property - def data_source(self): - """Look up the DataSource object corresponding to the - service we're running this data through. - - Out of an excess of caution, we look up the DataSource every - time, rather than storing it, in case a CoverageProvider is - ever used in an environment where the database session is - scoped (e.g. the circulation manager). - """ - return DataSource.lookup(self._db, self.DATA_SOURCE_NAME) - - def failure(self, identifier, error, transient=True): - """Create a CoverageFailure object to memorialize an error.""" - return CoverageFailure( - identifier, - error, - data_source=self.data_source, - transient=transient, - collection=self.collection_or_not, - ) - - def can_cover(self, identifier): - """Can this IdentifierCoverageProvider do anything with the given - Identifier? - - This is not needed in the normal course of events, but a - caller may need to decide whether to pass an Identifier - into ensure_coverage() or register(). - """ - return ( - not self.input_identifier_types - or identifier.type in self.input_identifier_types - ) - - def run_on_specific_identifiers(self, identifiers): - """Split a specific set of Identifiers into batches and process one - batch at a time. - - This is for use by IdentifierInputScript. - - :return: The same (counts, records) 2-tuple as - process_batch_and_handle_results. - """ - index = 0 - successes = 0 - transient_failures = 0 - persistent_failures = 0 - records = [] - - # Of all the items that need coverage, find the intersection - # with the given list of items. - need_coverage = self.items_that_need_coverage(identifiers).all() - - # Treat any items with up-to-date coverage records as - # automatic successes. - # - # NOTE: We won't actually be returning those coverage records - # in `records`, since items_that_need_coverage() filters them - # out, but nobody who calls this method really needs those - # records. - automatic_successes = len(identifiers) - len(need_coverage) - successes += automatic_successes - self.log.info("%d automatic successes.", successes) - - # Iterate over any items that were not automatic - # successes. - while index < len(need_coverage): - batch = need_coverage[index : index + self.batch_size] - (s, t, p), r = self.process_batch_and_handle_results(batch) - self.finalize_batch() - successes += s - transient_failures += t - persistent_failures += p - records += r - index += self.batch_size - return (successes, transient_failures, persistent_failures), records - - def ensure_coverage(self, item, force=False): - """Ensure coverage for one specific item. - - :param item: This should always be an Identifier, but this - code will also work if it's an Edition. (The Edition's - .primary_identifier will be covered.) - :param force: Run the coverage code even if an existing - coverage record for this item was created after `self.cutoff_time`. - :return: Either a coverage record or a CoverageFailure. - - TODO: This could be abstracted and moved to BaseCoverageProvider. - - """ - if isinstance(item, Identifier): - identifier = item - else: - identifier = item.primary_identifier - - if self.COVERAGE_COUNTS_FOR_EVERY_COLLECTION: - # We need to cover this Identifier once, and then we're - # done, for all collections. - collection = None - else: - # We need separate coverage for the specific Collection - # associated with this CoverageProvider. - collection = self.collection - - coverage_record = get_one( - self._db, - CoverageRecord, - identifier=identifier, - collection=collection, - data_source=self.data_source, - operation=self.operation, - on_multiple="interchangeable", - ) - if not force and not self.should_update(coverage_record): - return coverage_record - - counts, records = self.process_batch_and_handle_results([identifier]) - if records: - coverage_record = records[0] - else: - coverage_record = None - return coverage_record - - def edition(self, identifier): - """Finds or creates an Edition representing this coverage provider's - view of a given Identifier. - """ - edition, ignore = Edition.for_foreign_id( - self._db, self.data_source, identifier.type, identifier.identifier - ) - return edition - - def set_bibliographic(self, identifier, bibliographic): - """Finds or creates the Edition for an Identifier, updates it - with the given bibliographic data. - - :return: The Identifier (if successful) or an appropriate - CoverageFailure (if not). - """ - edition = self.edition(identifier) - if isinstance(edition, CoverageFailure): - return edition - - if not bibliographic: - e = "Did not receive bibliographic data from input source" - return self.failure(identifier, e, transient=True) - - try: - # We're passing in the Collection even if this - # CoverageProvider has - # COVERAGE_COUNTS_FOR_EVERY_COLLECTION set to False. If - # we did happen to get some circulation information while - # we were at it, we might as well store it properly. - # The data layer will not use the collection when creating - # CoverageRecords for the bibliographic actions. - bibliographic.apply( - self._db, - edition, - collection=self.collection, - replace=self.replacement_policy, - ) - except Exception as e: - self.log.warning( - "Error applying bibliographic data to edition %d: %s", - edition.id, - e, - exc_info=e, - ) - return self.failure(identifier, repr(e), transient=True) - - return identifier - - # - # Implementation of BaseCoverageProvider virtual methods. - # - - def items_that_need_coverage(self, identifiers=None, **kwargs): - """Find all items lacking coverage from this CoverageProvider. - - Items should be Identifiers, though Editions should also work. - - By default, all identifiers of the `INPUT_IDENTIFIER_TYPES` which - don't already have coverage are chosen. - - :param identifiers: The batch of identifier objects to test - for coverage. identifiers and self.input_identifiers can - intersect -- if this provider was created for the purpose - of running specific Identifiers, and within those - Identifiers you want to batch, you can use both - parameters. - """ - qu = Identifier.missing_coverage_from( - self._db, - self.input_identifier_types, - self.data_source, - count_as_missing_before=self.cutoff_time, - operation=self.operation, - identifiers=self.input_identifiers, - collection=self.collection_or_not, - **kwargs, - ) - - if identifiers: - qu = qu.filter(Identifier.id.in_([x.id for x in identifiers])) - if not identifiers and identifiers != None: - # An empty list was provided. The returned query should be empty. - qu = qu.filter(Identifier.id == None) - - if self.registered_only: - # Return Identifiers that have been "registered" for coverage - # or already have a failure from previous coverage attempts. - qu = qu.filter(CoverageRecord.id != None) - - return qu - - def add_coverage_record_for(self, item): - """Record this CoverageProvider's coverage for the given - Edition/Identifier, as a CoverageRecord. - """ - record, is_new = CoverageRecord.add_for( - item, - data_source=self.data_source, - operation=self.operation, - collection=self.collection_or_not, - ) - record.status = CoverageRecord.SUCCESS - record.exception = None - return record - - def record_failure_as_coverage_record(self, failure): - """Turn a CoverageFailure into a CoverageRecord object.""" - return failure.to_coverage_record(operation=self.operation) - - def failure_for_ignored_item(self, item): - """Create a CoverageFailure recording the CoverageProvider's - failure to even try to process an item. - """ - return self.failure(item, "Was ignored by CoverageProvider.", transient=True) - - -class CollectionCoverageProvider(IdentifierCoverageProvider): - """A CoverageProvider that covers all the Identifiers currently - licensed to a given Collection. - - You should subclass this CoverageProvider if you want to create - Works (as opposed to operating on existing Works) or update the - circulation information for LicensePools. You can't use it to - create new LicensePools, since it only operates on Identifiers - that already have a LicencePool in the given Collection. - - If a book shows up in multiple Collections, the first Collection - to process it takes care of it for the others. Any books that were - processed through their membership in another Collection will be - left alone. - - For this reason it's important that subclasses of this - CoverageProvider only deal with bibliographic information and - format availability information (such as links to open-access - downloads). You'll have problems if you try to use - CollectionCoverageProvider to keep track of information like the - number of licenses available for a book. - - In addition to defining the class variables defined by - CoverageProvider, you must define the class variable PROTOCOL when - subclassing this class. This is the entity that provides the - licenses for this Collection. It should be a valid protocol - in the LICENSE_GOAL registry. - """ - - # By default, this type of CoverageProvider will provide coverage to - # all Identifiers in the given Collection, regardless of their type. - INPUT_IDENTIFIER_TYPES: None | str | object = None - - DEFAULT_BATCH_SIZE = 10 - - # Set this to the name of the protocol managed by this type of - # CoverageProvider. If this CoverageProvider can manage collections - # for any protocol, leave this as None. - PROTOCOL: str | None = None - - # By default, Works calculated by a CollectionCoverageProvider update - # the ExternalSearchIndex. Set this value to True for applications that - # don't use external search, such as the Metadata Wrangler. - EXCLUDE_SEARCH_INDEX = False - - def __init__(self, collection, **kwargs): - """Constructor. - - :param collection: Will provide coverage to all Identifiers with - a LicensePool licensed to the given Collection. - """ - if not isinstance(collection, Collection): - raise CollectionMissing( - "%s must be instantiated with a Collection." % (self.__class__.__name__) - ) - - if self.PROTOCOL and collection.protocol != self.PROTOCOL: - raise ValueError( - "Collection protocol (%s) does not match CoverageProvider protocol (%s)" - % (collection.protocol, self.PROTOCOL) - ) - _db = Session.object_session(collection) - super().__init__(_db, collection, **kwargs) - - def _default_replacement_policy(self, _db): - """Unless told otherwise, assume that we are getting - this data from a reliable source of both bibliographic and circulation - information. - """ - return ReplacementPolicy.from_license_source() - - @classmethod - def collections(cls, _db): - """Returns a list of randomly sorted list of collections covered by the - provider. - """ - if cls.PROTOCOL: - collections = Collection.by_protocol(_db, cls.PROTOCOL) - else: - collections = _db.query(Collection) - return collections.order_by(func.random()).all() - - @classmethod - def all(cls, _db, **kwargs): - """Yield a sequence of CollectionCoverageProvider instances, one for - every Collection that gets its licenses from cls.PROTOCOL. - - CollectionCoverageProviders will be yielded in a random order. - - :param kwargs: Keyword arguments passed into the constructor for - CollectionCoverageProvider (or, more likely, one of its subclasses). - - """ - for collection in cls.collections(_db): - yield cls(collection, **kwargs) - - def run_once(self, *args, **kwargs): - self.log.info("Considering collection %s", self.collection.name) - return super().run_once(*args, **kwargs) - - def items_that_need_coverage(self, identifiers=None, **kwargs): - """Find all Identifiers associated with this Collection but lacking - coverage through this CoverageProvider. - """ - qu = super().items_that_need_coverage(identifiers, **kwargs) - qu = qu.join(Identifier.licensed_through).filter( - LicensePool.collection_id == self.collection_id - ) - return qu - - def license_pool(self, identifier, data_source=None): - """Finds this Collection's LicensePool for the given Identifier, - creating one if necessary. - - :param data_source: If it's necessary to create a LicensePool, - the new LicensePool will have this DataSource. The default is to - use the DataSource associated with the CoverageProvider. This - should only be needed by the metadata wrangler. - """ - license_pools = [ - p for p in identifier.licensed_through if self.collection == p.collection - ] - - if license_pools: - # A given Collection may have at most one LicensePool for - # a given identifier. - return license_pools[0] - - data_source = data_source or self.data_source - if isinstance(data_source, (bytes, str)): - data_source = DataSource.lookup(self._db, data_source) - - # This Collection has no LicensePool for the given Identifier. - # Create one. - # - # Under normal circumstances, this will never happen, because - # CollectionCoverageProvider only operates on Identifiers that - # already have a LicensePool in this Collection. - # - # However, this does happen in the metadata wrangler, - # which typically has to manage information about books it has no - # rights to. - license_pool, ignore = LicensePool.for_foreign_id( - self._db, - data_source, - identifier.type, - identifier.identifier, - collection=self.collection, - ) - return license_pool - - def work(self, identifier, license_pool=None, **calculate_work_kwargs): - """Finds or creates a Work for this Identifier as licensed through - this Collection. - - If the given Identifier already has a Work associated with it, - that Work will always be used, since an Identifier can only have one - Work associated with it. - - However, if there is no current Work, a Work will only be - created if the given Identifier already has a LicensePool in - the Collection associated with this CoverageProvider (or if a - LicensePool to use is provided.) This method will not create - new LicensePools. - - If the work is newly created or an existing work is not - presentation-ready, a new Work will be created by calling - LicensePool.calculate_work(). If there is an existing - presentation-ready work, calculate_work() will not be called; - instead, the work will be slated for recalculation when its - metadata changes through BibliographicData.apply(). - - :param calculate_work_kwargs: Keyword arguments to pass into - calculate_work() if and when it is called. - - :return: A Work, if possible. Otherwise, a CoverageFailure explaining - why no Work could be created. - - """ - work = identifier.work - if work and work.presentation_ready: - # There is already a presentation-ready Work associated - # with this Identifier. Return it. - return work - - # There is no presentation-ready Work associated with this - # Identifier. We need to create one, if possible. - error = None - if not license_pool: - license_pool, ignore = LicensePool.for_foreign_id( - self._db, - self.data_source, - identifier.type, - identifier.identifier, - collection=self.collection, - autocreate=False, - ) - - if license_pool: - if not license_pool.work or not license_pool.work.presentation_ready: - for v, default in (("exclude_search", self.EXCLUDE_SEARCH_INDEX),): - if not v in calculate_work_kwargs: - calculate_work_kwargs[v] = default - - # Calling calculate_work will calculate the work's - # presentation and make it presentation-ready if - # possible. - work, created = license_pool.calculate_work(**calculate_work_kwargs) - if not work: - error = "Work could not be calculated" - else: - error = "Cannot locate LicensePool" - - if error: - return self.failure(identifier, error, transient=True) - return work - - def set_bibliographic_and_circulation_data( - self, - identifier, - bibliographicdata, - circulationdata, - ): - """Makes sure that the given Identifier has a Work, Edition (in the - context of this Collection), and LicensePool (ditto), and that - all the information is up to date. - - :return: The Identifier (if successful) or an appropriate - CoverageFailure (if not). - """ - - if not bibliographicdata and not circulationdata: - e = "Received neither bibliographic data nor circulation data from input source" - return self.failure(identifier, e, transient=True) - - if bibliographicdata: - result = self.set_bibliographic(identifier, bibliographicdata) - if isinstance(result, CoverageFailure): - return result - - if circulationdata: - result = self._set_circulationdata(identifier, circulationdata) - if isinstance(result, CoverageFailure): - return result - - # By this point the Identifier should have an appropriate - # Edition and LicensePool. We should now be able to make a - # Work. - work = self.work(identifier) - if isinstance(work, CoverageFailure): - return work - - return identifier - - def _set_circulationdata(self, identifier, circulationdata): - """Finds or creates a LicensePool for an Identifier, updates it - with the given circulationdata, then creates a Work for the book. - - :return: The Identifier (if successful) or an appropriate - CoverageFailure (if not). - """ - error = None - if circulationdata: - try: - primary_identifier = circulationdata.load_primary_identifier(self._db) - except ValueError: - primary_identifier = None - if identifier != primary_identifier: - error = "Identifier did not match CirculationData's primary identifier." - else: - error = "Did not receive circulationdata from input source" - - if error: - return self.failure(identifier, error, transient=True) - - try: - circulationdata.apply( - self._db, self.collection, replace=self.replacement_policy - ) - except Exception as e: - if self.collection: - collection_name = " to collection %s" % self.collection.name - else: - collection_name = "" - self.log.warning( - "Error applying circulationdata%s: %s", collection_name, e, exc_info=e - ) - return self.failure(identifier, repr(e), transient=True) - - return identifier - - def set_presentation_ready(self, identifier): - """Set a Work presentation-ready.""" - work = self.work(identifier) - if isinstance(work, CoverageFailure): - return work - work.set_presentation_ready(exclude_search=self.EXCLUDE_SEARCH_INDEX) - return identifier - - -class BibliographicCoverageProvider(CollectionCoverageProvider): - """Fill in bibliographic metadata for all books in a Collection. - - e.g. ensures that we get Overdrive coverage for all Overdrive IDs - in a collection. - - Although a BibliographicCoverageProvider may gather - CirculationData for a book, it cannot guarantee equal coverage for - all Collections that contain that book. CirculationData should be - limited to things like formats that don't vary between - Collections, and you should use a CollectionMonitor to make sure - your circulation information is up-to-date for each Collection. - """ - - def handle_success(self, identifier): - """Once a book has bibliographic coverage, it can be given a - work and made presentation ready. - """ - self.set_presentation_ready(identifier) diff --git a/src/palace/manager/data_layer/bibliographic.py b/src/palace/manager/data_layer/bibliographic.py index 2346927a34..6b5ff6b50a 100644 --- a/src/palace/manager/data_layer/bibliographic.py +++ b/src/palace/manager/data_layer/bibliographic.py @@ -24,7 +24,6 @@ from palace.manager.sqlalchemy.model.classification import Classification from palace.manager.sqlalchemy.model.collection import Collection from palace.manager.sqlalchemy.model.contributor import Contributor -from palace.manager.sqlalchemy.model.coverage import CoverageRecord from palace.manager.sqlalchemy.model.datasource import DataSource from palace.manager.sqlalchemy.model.edition import Edition from palace.manager.sqlalchemy.model.identifier import Identifier @@ -784,8 +783,6 @@ def apply( edition: Edition, collection: Collection | None, replace: ReplacementPolicy | None = None, - *, - create_coverage_record: bool = True, ) -> tuple[Edition, bool]: """Apply full bibliographic metadata (identifier + edition) and calculate work presentation. @@ -801,8 +798,6 @@ def apply( :param edition: The Edition object to update with bibliographic data. :param collection: Optional Collection for applying circulation data. :param replace: Policy controlling which fields to replace. Defaults to ReplacementPolicy(). - :param create_coverage_record: If True, creates/updates a CoverageRecord for this edition. - Will be removed once coverage records are fully deprecated. :return: Tuple of (updated_edition, requires_presentation_update) where the boolean indicates whether the edition or its work requires presentation recalculation due to changes in core fields (title, contributors, images, descriptions, etc.). @@ -868,18 +863,6 @@ def apply( if self.circulation: self.circulation.apply(db, collection, replace) - # Update the coverage record for this edition and data - # source. We omit the collection information, even if we know - # which collection this is, because we only changed bibliographic data. - # TODO: Remove this once we have done away with coverage records - if create_coverage_record: - CoverageRecord.add_for( - edition, - data_source, - timestamp=self.as_of_timestamp, - collection=None, - ) - # Calculate work presentation directly if needed if work_requires_full_recalculation or work_requires_new_presentation_edition: pool = get_one( diff --git a/src/palace/manager/integration/license/bibliotheca.py b/src/palace/manager/integration/license/bibliotheca.py index aaa1765607..654cc14cc3 100644 --- a/src/palace/manager/integration/license/bibliotheca.py +++ b/src/palace/manager/integration/license/bibliotheca.py @@ -60,7 +60,6 @@ CannotLoadConfiguration, ConfigurationAttributeValue, ) -from palace.manager.core.coverage import BibliographicCoverageProvider, CoverageFailure from palace.manager.core.selftest import SelfTestResult from palace.manager.data_layer.bibliographic import BibliographicData from palace.manager.data_layer.circulation import CirculationData @@ -1219,56 +1218,3 @@ def process_one( end_time, internal_event_type, ) - - -BibliothecaApiClassT = type[BibliothecaAPI] | BibliothecaAPI - - -class BibliothecaBibliographicCoverageProvider(BibliographicCoverageProvider): - """Fill in bibliographic metadata for Bibliotheca records. - - This will occasionally fill in some availability information for a - single Collection, but we rely on Monitors to keep availability - information up to date for all Collections. - """ - - SERVICE_NAME = "Bibliotheca Bibliographic Coverage Provider" - DATA_SOURCE_NAME = DataSource.BIBLIOTHECA - PROTOCOL = BibliothecaAPI.label() - INPUT_IDENTIFIER_TYPES = Identifier.BIBLIOTHECA_ID - - # 25 is the maximum batch size for the Bibliotheca API. - DEFAULT_BATCH_SIZE = 25 - - def __init__( - self, - collection: Collection, - api_class: BibliothecaApiClassT = BibliothecaAPI, - **kwargs: Any, - ) -> None: - """Constructor. - - :param collection: Provide bibliographic coverage to all - Bibliotheca books in the given Collection. - :param api_class: Instantiate this class with the given Collection, - rather than instantiating BibliothecaAPI. - :param input_identifiers: Passed in by RunCoverageProviderScript. - A list of specific identifiers to get coverage for. - """ - super().__init__(collection, **kwargs) - if isinstance(api_class, BibliothecaAPI): - # This is an already instantiated API object. Use it - # instead of creating a new one. - self.api = api_class - else: - # A web application should not use this option because it - # will put a non-scoped session in the mix. - _db = Session.object_session(collection) - self.api = api_class(_db, collection) - - def process_item(self, identifier: Identifier) -> Identifier | CoverageFailure: - list_bibliographic = self.api.bibliographic_lookup(identifier) - if not list_bibliographic: - return self.failure(identifier, "Bibliotheca bibliographic lookup failed.") # type: ignore[no-any-return] - [bibliographic] = list_bibliographic - return self.set_bibliographic(identifier, bibliographic) # type: ignore[no-any-return] diff --git a/src/palace/manager/integration/license/bibliotheca_circulation_updater.py b/src/palace/manager/integration/license/bibliotheca_circulation_updater.py index d89e7ba696..9f315c12ab 100644 --- a/src/palace/manager/integration/license/bibliotheca_circulation_updater.py +++ b/src/palace/manager/integration/license/bibliotheca_circulation_updater.py @@ -222,7 +222,6 @@ def _process_batch( edition, self._collection, ReplacementPolicy.from_license_source(), - create_coverage_record=False, ) else: apply.bibliographic_apply.delay( diff --git a/src/palace/manager/integration/license/overdrive/api.py b/src/palace/manager/integration/license/overdrive/api.py index 0b1c152221..03f3691ce7 100644 --- a/src/palace/manager/integration/license/overdrive/api.py +++ b/src/palace/manager/integration/license/overdrive/api.py @@ -69,9 +69,6 @@ OVERDRIVE_STREAMING_FORMATS, OverdriveConstants, ) -from palace.manager.integration.license.overdrive.coverage import ( - OverdriveBibliographicCoverageProvider, -) from palace.manager.integration.license.overdrive.exception import ( InvalidFieldOptionError, OverdriveModelError, @@ -341,9 +338,6 @@ def __init__(self, _db: Session, collection: Collection) -> None: # instances (e.g. Flask workers) transparently re-fetch a rotated token # without a process restart. See the collection_token property. self._cached_collection_token: OverdriveToken | None = None - self.overdrive_bibliographic_coverage_provider = ( - OverdriveBibliographicCoverageProvider(collection, api=self) - ) @property def settings(self) -> OverdriveSettings: @@ -1782,10 +1776,9 @@ def update_licensepool( ) if is_new or not license_pool.work: # Either this is the first time we've seen this book or it doesn't - # have an associated work. Make sure its identifier has bibliographic coverage. - self.overdrive_bibliographic_coverage_provider.ensure_coverage( - license_pool.identifier, force=True - ) + # have an associated work. Make sure its identifier has bibliographic + # coverage and a presentation-ready work. + self._ensure_bibliographic_coverage(license_pool) return self.update_licensepool_with_book_info( availability, resolved_book_id, license_pool, is_new @@ -1806,6 +1799,48 @@ def _edition(self, licensepool: LicensePool) -> tuple[Edition, bool]: licensepool.identifier.identifier, ) + def _ensure_bibliographic_coverage(self, license_pool: LicensePool) -> None: + """Fetch Overdrive bibliographic metadata, apply it to the Edition, and + make the LicensePool's Work presentation-ready. + + This replaces the former ``OverdriveBibliographicCoverageProvider`` + path that ran inside :meth:`update_licensepool`. A bad or unrecognized + Overdrive ID simply leaves the pool without a Work, matching the prior + behavior (``update_licensepool`` ignored the coverage result). + """ + identifier = license_pool.identifier + info = self.metadata_lookup(identifier) + if info.get("errorCode") in ("NotFound", "InvalidGuid"): + self.log.warning( + "Could not get Overdrive metadata for %s: %s", + identifier.identifier, + info.get("errorCode"), + ) + return + + bibliographic = OverdriveRepresentationExtractor.book_info_to_bibliographic( + info + ) + if not bibliographic: + self.log.warning( + "Could not extract bibliographic data from Overdrive metadata for %s.", + identifier.identifier, + ) + return + + edition, _ = self._edition(license_pool) + bibliographic.apply( + self._db, + edition, + self.collection, + replace=ReplacementPolicy.from_license_source(), + ) + + if not license_pool.work or not license_pool.work.presentation_ready: + work, _ = license_pool.calculate_work() + if work: + work.set_presentation_ready() + def update_licensepool_with_book_info( self, availability: Availability, diff --git a/src/palace/manager/integration/license/overdrive/coverage.py b/src/palace/manager/integration/license/overdrive/coverage.py deleted file mode 100644 index 508797a562..0000000000 --- a/src/palace/manager/integration/license/overdrive/coverage.py +++ /dev/null @@ -1,85 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING, Any - -from sqlalchemy.orm import Session - -from palace.manager.core.coverage import BibliographicCoverageProvider, CoverageFailure -from palace.manager.data_layer.bibliographic import BibliographicData -from palace.manager.integration.license.overdrive.constants import OVERDRIVE_LABEL -from palace.manager.integration.license.overdrive.representation import ( - OverdriveRepresentationExtractor, -) -from palace.manager.sqlalchemy.model.collection import Collection -from palace.manager.sqlalchemy.model.datasource import DataSource -from palace.manager.sqlalchemy.model.identifier import Identifier - -if TYPE_CHECKING: - from palace.manager.integration.license.overdrive.api import OverdriveAPI - - -class OverdriveBibliographicCoverageProvider(BibliographicCoverageProvider): - """Fill in bibliographic metadata for Overdrive records. - - This will occasionally fill in some availability information for a - single Collection, but we rely on Monitors to keep availability - information up to date for all Collections. - """ - - SERVICE_NAME = "Overdrive Bibliographic Coverage Provider" - DATA_SOURCE_NAME = DataSource.OVERDRIVE - PROTOCOL = OVERDRIVE_LABEL - INPUT_IDENTIFIER_TYPES = Identifier.OVERDRIVE_ID - - def __init__( - self, - collection: Collection, - api: OverdriveAPI | None = None, - **kwargs: Any, - ) -> None: - """Constructor. - - :param collection: Provide bibliographic coverage to all - Overdrive books in the given Collection. - :param api: API class, if none it will be initialized with OverdriveAPI. - """ - super().__init__(collection, **kwargs) - if api is None: - # A web application should not use this option because it - # will put a non-scoped session in the mix. - _db = Session.object_session(collection) - from palace.manager.integration.license.overdrive.api import OverdriveAPI - - self.api = OverdriveAPI(_db, collection) - else: - self.api = api - - def process_item(self, identifier: Identifier) -> Identifier | CoverageFailure: - info = self.api.metadata_lookup(identifier) - error = None - if info.get("errorCode") == "NotFound": - error = "ID not recognized by Overdrive: %s" % identifier.identifier - elif info.get("errorCode") == "InvalidGuid": - error = "Invalid Overdrive ID: %s" % identifier.identifier - - if error: - return self.failure(identifier, error, transient=False) # type: ignore[no-any-return] - - bibliographic = OverdriveRepresentationExtractor.book_info_to_bibliographic( - info - ) - - if not bibliographic: - e = "Could not extract bibliographic data from Overdrive data: %r" % info - return self.failure(identifier, e) # type: ignore[no-any-return] - - self.bibliographic_data_pre_hook(bibliographic) - return self.set_bibliographic(identifier, bibliographic) # type: ignore[no-any-return] - - def bibliographic_data_pre_hook( - self, bibliographic: BibliographicData - ) -> BibliographicData: - """A hook method that allows subclasses to modify a BibliographicData - object derived from Overdrive before it's applied. - """ - return bibliographic diff --git a/src/palace/manager/scripts/coverage_provider.py b/src/palace/manager/scripts/coverage_provider.py deleted file mode 100644 index d1c951fd8e..0000000000 --- a/src/palace/manager/scripts/coverage_provider.py +++ /dev/null @@ -1,169 +0,0 @@ -import argparse -import random -import sys -from collections.abc import Callable, Sequence -from typing import Any - -from sqlalchemy.orm import Session - -from palace.manager.core.coverage import ( - BaseCoverageProvider, - CollectionCoverageProvider, - CoverageProviderProgress, - IdentifierCoverageProvider, -) -from palace.manager.scripts.base import Script, _normalize_cmd_args -from palace.manager.scripts.input import IdentifierInputScript, SupportsReadlines -from palace.manager.sqlalchemy.model.identifier import Identifier - - -class RunCoverageProvidersScript(Script): - """Alternate between multiple coverage providers.""" - - def __init__( - self, - providers: Sequence[ - BaseCoverageProvider | Callable[[Session], BaseCoverageProvider] - ], - _db: Session | None = None, - ) -> None: - super().__init__(_db=_db) - self.providers = [] - for i in providers: - if callable(i): - i = i(self._db) - self.providers.append(i) - - def do_run(self) -> list[CoverageProviderProgress | None]: - providers = list(self.providers) - if not providers: - self.log.info("No CoverageProviders to run.") - - progress: list[CoverageProviderProgress | None] = [] - while providers: - random.shuffle(providers) - for provider in providers: - self.log.debug("Running %s", provider.service_name) - - try: - provider_progress = provider.run_once_and_update_timestamp() - progress.append(provider_progress) - except Exception as e: - self.log.error( - "Error in %r, moving on to next CoverageProvider.", - provider, - exc_info=e, - ) - - self.log.debug("Completed %s", provider.service_name) - providers.remove(provider) - return progress - - -class RunCollectionCoverageProviderScript(RunCoverageProvidersScript): - """Run the same CoverageProvider code for all Collections that - get their licenses from the appropriate place. - """ - - def __init__( - self, - provider_class: type[CollectionCoverageProvider] | None, - _db: Session | None = None, - providers: Sequence[BaseCoverageProvider] | None = None, - **kwargs: Any, - ) -> None: - _db = _db or self._db - providers = list(providers or []) - if provider_class: - providers += self.get_providers(_db, provider_class, **kwargs) - super().__init__(providers, _db=_db) - - def get_providers( - self, - _db: Session, - provider_class: type[CollectionCoverageProvider], - **kwargs: Any, - ) -> list[BaseCoverageProvider]: - return list(provider_class.all(_db, **kwargs)) - - -class RunCoverageProviderScript(IdentifierInputScript): - """Run a single coverage provider.""" - - @classmethod - def arg_parser(cls, _db: Session) -> argparse.ArgumentParser: - parser = IdentifierInputScript.arg_parser(_db) - parser.add_argument( - "--cutoff-time", - help="Update existing coverage records if they were originally created after this time.", - ) - return parser - - @classmethod - def parse_command_line( - cls, - _db: Session, - cmd_args: Sequence[str | None] | None = None, - stdin: SupportsReadlines = sys.stdin, - *args: Any, - **kwargs: Any, - ) -> argparse.Namespace: - parser = cls.arg_parser(_db) - parsed = parser.parse_args(_normalize_cmd_args(cmd_args)) - stdin_lines = cls.read_stdin_lines(stdin) - parsed = cls.look_up_identifiers(_db, parsed, stdin_lines, *args, **kwargs) - if parsed.cutoff_time: - parsed.cutoff_time = cls.parse_time(parsed.cutoff_time) - return parsed - - def __init__( - self, - provider: ( - IdentifierCoverageProvider | Callable[..., IdentifierCoverageProvider] - ), - _db: Session | None = None, - cmd_args: Sequence[str | None] | None = None, - *provider_args: Any, - **provider_kwargs: Any, - ) -> None: - super().__init__(_db) - parsed_args = self.parse_command_line(self._db, cmd_args) - if parsed_args.identifier_type: - self.identifier_type = parsed_args.identifier_type - self.identifier_types = [self.identifier_type] - else: - self.identifier_type = None - self.identifier_types = [] - - if parsed_args.identifiers: - self.identifiers: list[Identifier] = parsed_args.identifiers - else: - self.identifiers = [] - - if callable(provider): - kwargs = self.extract_additional_command_line_arguments() - kwargs.update(provider_kwargs) - - provider = provider( - self._db, *provider_args, cutoff_time=parsed_args.cutoff_time, **kwargs - ) - self.provider = provider - self.name = self.provider.service_name - - def extract_additional_command_line_arguments(self) -> dict[str, list[Identifier]]: - """A hook method for subclasses. - - Turns command-line arguments into additional keyword arguments - to the CoverageProvider constructor. - - By default, pass in a value used only by CoverageProvider - """ - return { - "input_identifiers": self.identifiers, - } - - def do_run(self) -> None: - if self.identifiers: - self.provider.run_on_specific_identifiers(self.identifiers) - else: - self.provider.run() diff --git a/src/palace/manager/scripts/informational.py b/src/palace/manager/scripts/informational.py index c043301e39..50c964e3a7 100644 --- a/src/palace/manager/scripts/informational.py +++ b/src/palace/manager/scripts/informational.py @@ -22,8 +22,6 @@ from palace.manager.search.filter import Filter from palace.manager.sqlalchemy.model.circulationevent import CirculationEvent from palace.manager.sqlalchemy.model.collection import Collection -from palace.manager.sqlalchemy.model.coverage import CoverageRecord -from palace.manager.sqlalchemy.model.datasource import DataSource from palace.manager.sqlalchemy.model.edition import Edition from palace.manager.sqlalchemy.model.identifier import Identifier from palace.manager.sqlalchemy.model.integration import IntegrationConfiguration @@ -366,12 +364,6 @@ def explain_identifier( self.explain_identifier( output, False, seen, equivalency.strength, level + 1 ) - if primary: - crs = identifier.coverage_records - if crs: - self.write(" %d coverage records:" % len(crs)) - for cr in sorted(crs, key=lambda x: x.timestamp or datetime.min): - self.explain_coverage_record(cr) def explain_license_pool(self, pool: LicensePool) -> None: self.write("Licensepool info:") @@ -435,46 +427,6 @@ def explain_work(self, work: Work) -> None: collection = "!collection" self.write(f" ACTIVE: {pool.identifier!r} {collection}") - def explain_coverage_record(self, cr: CoverageRecord) -> None: - if cr.timestamp is None: - return - status_value = str(cr.status) if cr.status is not None else None - operation_value = str(cr.operation) if cr.operation is not None else None - self._explain_coverage_record( - cr.timestamp, cr.data_source, operation_value, status_value, cr.exception - ) - - def _explain_coverage_record( - self, - timestamp: datetime, - data_source: DataSource | None, - operation: str | None, - status: str | None, - exception: str | None, - ) -> None: - timestamp_string = timestamp.strftime(self.TIME_FORMAT) - if data_source: - data_source_value = data_source.name + " | " - else: - data_source_value = "" - if operation: - operation_value = operation + " | " - else: - operation_value = "" - if exception: - exception_value = " | " + exception - else: - exception_value = "" - self.write( - " {} | {}{}{}{}".format( - timestamp_string, - data_source_value, - operation_value, - status, - exception_value, - ) - ) - class WhereAreMyBooksScript(CollectionInputScript): """Try to figure out why Works aren't showing up. diff --git a/src/palace/manager/sqlalchemy/model/collection.py b/src/palace/manager/sqlalchemy/model/collection.py index 1e854cd2b2..828b094d24 100644 --- a/src/palace/manager/sqlalchemy/model/collection.py +++ b/src/palace/manager/sqlalchemy/model/collection.py @@ -34,7 +34,7 @@ from palace.manager.sqlalchemy.hassessioncache import HasSessionCache from palace.manager.sqlalchemy.hybrid import hybrid_property from palace.manager.sqlalchemy.model.base import Base -from palace.manager.sqlalchemy.model.coverage import CoverageRecord, Timestamp +from palace.manager.sqlalchemy.model.coverage import Timestamp from palace.manager.sqlalchemy.model.datasource import DataSource from palace.manager.sqlalchemy.model.identifier import Identifier from palace.manager.sqlalchemy.model.integration import ( @@ -133,12 +133,6 @@ class Collection(Base, HasSessionCache, RedisKeyMixin): "Identifier", secondary="collections_identifiers", back_populates="collections" ) - # A Collection can be associated with multiple CoverageRecords - # for Identifiers in its catalog. - coverage_records: Mapped[list[CoverageRecord]] = relationship( - "CoverageRecord", back_populates="collection", cascade="all" - ) - # A collection may be associated with one or more custom lists. # When a new license pool is added to the collection, it will # also be added to the list. Admins can remove items from the diff --git a/src/palace/manager/sqlalchemy/model/coverage.py b/src/palace/manager/sqlalchemy/model/coverage.py index 05273239c2..56a289a58c 100644 --- a/src/palace/manager/sqlalchemy/model/coverage.py +++ b/src/palace/manager/sqlalchemy/model/coverage.py @@ -1,4 +1,4 @@ -# BaseCoverageRecord, Timestamp, CoverageRecord +# BaseCoverageRecord, Timestamp from __future__ import annotations import datetime @@ -11,7 +11,6 @@ DateTime, Enum, ForeignKey, - Index, Integer, String, Unicode, @@ -19,7 +18,7 @@ ) from sqlalchemy.orm import Mapped, relationship from sqlalchemy.orm.session import Session -from sqlalchemy.sql.expression import and_, literal, literal_column, or_ +from sqlalchemy.sql.expression import or_ from palace.util.datetime_helpers import utc_now @@ -29,8 +28,6 @@ if TYPE_CHECKING: from palace.manager.sqlalchemy.model.collection import Collection - from palace.manager.sqlalchemy.model.datasource import DataSource - from palace.manager.sqlalchemy.model.identifier import Identifier class BaseCoverageRecord: @@ -357,330 +354,3 @@ def recording(self) -> Generator[Self]: self.finish = utc_now() __table_args__ = (UniqueConstraint("service", "collection_id"),) - - -class CoverageRecord(Base, BaseCoverageRecord): - """A record of a Identifier being used as input into some process.""" - - __tablename__ = "coveragerecords" - - REAP_OPERATION = "reap" - IMPORT_OPERATION = "import" - RESOLVE_IDENTIFIER_OPERATION = "resolve-identifier" - REPAIR_SORT_NAME_OPERATION = "repair-sort-name" - METADATA_UPLOAD_OPERATION = "metadata-upload" - - id: Mapped[int] = Column(Integer, primary_key=True) - identifier_id = Column(Integer, ForeignKey("identifiers.id"), index=True) - identifier: Mapped[Identifier | None] = relationship( - "Identifier", back_populates="coverage_records" - ) - - # If applicable, this is the ID of the data source that took the - # Identifier as input. - data_source_id = Column(Integer, ForeignKey("datasources.id")) - data_source: Mapped[DataSource | None] = relationship( - "DataSource", back_populates="coverage_records" - ) - operation = Column(String(255), default=None) - - timestamp = Column(DateTime(timezone=True), index=True) - - status = Column(BaseCoverageRecord.status_enum, index=True) - exception = Column(Unicode, index=True) - - # If applicable, this is the ID of the collection for which - # coverage has taken place. This is currently only applicable - # for Metadata Wrangler coverage. - collection_id = Column(Integer, ForeignKey("collections.id"), nullable=True) - collection: Mapped[Collection | None] = relationship( - "Collection", back_populates="coverage_records" - ) - - __table_args__ = ( - Index( - "ix_identifier_id_data_source_id_operation", - identifier_id, - data_source_id, - operation, - unique=True, - postgresql_where=collection_id.is_(None), - ), - Index( - "ix_identifier_id_data_source_id_operation_collection_id", - identifier_id, - data_source_id, - operation, - collection_id, - unique=True, - ), - ) - - def __repr__(self): - template = '' - return self.human_readable(template) - - def human_readable(self, template): - """Interpolate data into a human-readable template.""" - if self.operation: - operation = ' operation="%s"' % self.operation - else: - operation = "" - if self.exception: - exception = ' exception="%s"' % self.exception - else: - exception = "" - return template % dict( - timestamp=self.timestamp.strftime("%Y-%m-%d %H:%M:%S"), - identifier_type=self.identifier.type, - identifier=self.identifier.identifier, - data_source=self.data_source.name, - operation=operation, - status=self.status, - exception=exception, - ) - - @classmethod - def assert_coverage_operation(cls, operation, collection): - if operation == CoverageRecord.IMPORT_OPERATION and not collection: - raise ValueError( - "An 'import' type coverage must be associated with a collection" - ) - - @classmethod - def lookup( - cls, edition_or_identifier, data_source, operation=None, collection=None - ): - from palace.manager.sqlalchemy.model.datasource import DataSource - from palace.manager.sqlalchemy.model.edition import Edition - from palace.manager.sqlalchemy.model.identifier import Identifier - - cls.assert_coverage_operation(operation, collection) - - _db = Session.object_session(edition_or_identifier) - if isinstance(edition_or_identifier, Identifier): - identifier = edition_or_identifier - elif isinstance(edition_or_identifier, Edition): - identifier = edition_or_identifier.primary_identifier - else: - raise ValueError( - "Cannot look up a coverage record for %r." % edition_or_identifier - ) - - if isinstance(data_source, (bytes, str)): - data_source = DataSource.lookup(_db, data_source) - - return get_one( - _db, - CoverageRecord, - identifier=identifier, - data_source=data_source, - operation=operation, - collection=collection, - on_multiple="interchangeable", - ) - - @classmethod - def add_for( - cls, - edition, - data_source, - operation=None, - timestamp=None, - status=BaseCoverageRecord.SUCCESS, - collection=None, - ): - from palace.manager.sqlalchemy.model.edition import Edition - from palace.manager.sqlalchemy.model.identifier import Identifier - - cls.assert_coverage_operation(operation, collection) - - _db = Session.object_session(edition) - if isinstance(edition, Identifier): - identifier = edition - elif isinstance(edition, Edition): - identifier = edition.primary_identifier - else: - raise ValueError("Cannot create a coverage record for %r." % edition) - timestamp = timestamp or utc_now() - coverage_record, is_new = get_one_or_create( - _db, - CoverageRecord, - identifier=identifier, - data_source=data_source, - operation=operation, - collection=collection, - on_multiple="interchangeable", - ) - coverage_record.status = status - coverage_record.timestamp = timestamp - return coverage_record, is_new - - @classmethod - def bulk_add( - cls, - identifiers, - data_source, - operation=None, - timestamp=None, - status=BaseCoverageRecord.SUCCESS, - exception=None, - collection=None, - force=False, - ): - """Create and update CoverageRecords so that every Identifier in - `identifiers` has an identical record. - """ - from palace.manager.sqlalchemy.model.identifier import Identifier - - if not identifiers: - # Nothing to do. - return - - cls.assert_coverage_operation(operation, collection) - - _db = Session.object_session(identifiers[0]) - timestamp = timestamp or utc_now() - identifier_ids = [i.id for i in identifiers] - - equivalent_record = and_( - cls.operation == operation, - cls.data_source == data_source, - cls.collection == collection, - ) - - updated_or_created_results = list() - if force: - # Make sure that works that previously had a - # CoverageRecord for this operation have their timestamp - # and status updated. - update = ( - cls.__table__.update() - .where( - and_( - cls.identifier_id.in_(identifier_ids), - equivalent_record, - ) - ) - .values(dict(timestamp=timestamp, status=status, exception=exception)) - .returning(cls.id, cls.identifier_id) - ) - updated_or_created_results = _db.execute(update).fetchall() - - already_covered = ( - _db.query(cls.id, cls.identifier_id) - .filter( - equivalent_record, - cls.identifier_id.in_(identifier_ids), - ) - .subquery() - ) - - # Make sure that any identifiers that need a CoverageRecord get one. - # The SELECT part of the INSERT...SELECT query. - data_source_id = data_source.id - collection_id = None - if collection: - collection_id = collection.id - - new_records = ( - _db.query( - Identifier.id.label("identifier_id"), - literal(operation, type_=String(255)).label("operation"), - literal(timestamp, type_=DateTime).label("timestamp"), - literal(status, type_=BaseCoverageRecord.status_enum).label("status"), - literal(exception, type_=Unicode).label("exception"), - literal(data_source_id, type_=Integer).label("data_source_id"), - literal(collection_id, type_=Integer).label("collection_id"), - ) - .select_from(Identifier) - .outerjoin( - already_covered, - Identifier.id == already_covered.c.identifier_id, - ) - .filter(already_covered.c.id == None) - ) - - new_records = new_records.filter(Identifier.id.in_(identifier_ids)) - - # The INSERT part. - insert = ( - cls.__table__.insert() - .from_select( - [ - literal_column("identifier_id"), - literal_column("operation"), - literal_column("timestamp"), - literal_column("status"), - literal_column("exception"), - literal_column("data_source_id"), - literal_column("collection_id"), - ], - new_records, - ) - .returning(cls.id, cls.identifier_id) - ) - - inserts = _db.execute(insert).fetchall() - - updated_or_created_results.extend(inserts) - _db.commit() - - # Default return for the case when all of the identifiers were - # ignored. - new_records = list() - ignored_identifiers = identifiers - - new_and_updated_record_ids = [r[0] for r in updated_or_created_results] - impacted_identifier_ids = [r[1] for r in updated_or_created_results] - - if new_and_updated_record_ids: - new_records = ( - _db.query(cls).filter(cls.id.in_(new_and_updated_record_ids)).all() - ) - - ignored_identifiers = [ - i for i in identifiers if i.id not in impacted_identifier_ids - ] - - return new_records, ignored_identifiers - - -Index( - "ix_coveragerecords_data_source_id_operation_identifier_id", - CoverageRecord.data_source_id, - CoverageRecord.operation, - CoverageRecord.identifier_id, -) - - -class EquivalencyCoverageRecord(Base, BaseCoverageRecord): - """Dormant model retained only so the ``equivalentscoveragerecords`` table - stays in the schema for one more release. - - The equivalent-identifiers refresh no longer reads or writes this table — it - was replaced by the Redis dirty-set queue and the ``equivalent_identifiers_refresh`` - Celery task. But per our online-migration convention the table cannot be dropped - in the same release that stops using it: during a rolling deploy, N-1 app servers - still run the old listener that writes here, so dropping the table now would make - them error. The table and this model will be removed in a follow-up PR that ships - after this release. See https://github.com/ThePalaceProject/circulation/pull/3459. - """ - - __tablename__ = "equivalentscoveragerecords" - - id: Mapped[int] = Column(Integer, primary_key=True) - - equivalency_id: Mapped[int] = Column( - Integer, - ForeignKey("equivalents.id", ondelete="CASCADE"), - index=True, - nullable=False, - ) - - operation = Column(String(255), index=True, default=None) - timestamp = Column(DateTime(timezone=True), index=True) - status = Column(BaseCoverageRecord.status_enum, index=True) - exception = Column(Unicode) - - __table_args__ = (UniqueConstraint(equivalency_id, operation),) diff --git a/src/palace/manager/sqlalchemy/model/datasource.py b/src/palace/manager/sqlalchemy/model/datasource.py index 96f40d2385..89fb2a7851 100644 --- a/src/palace/manager/sqlalchemy/model/datasource.py +++ b/src/palace/manager/sqlalchemy/model/datasource.py @@ -17,7 +17,6 @@ # This is needed during type checking so we have the # types of related models. from palace.manager.sqlalchemy.model.classification import Classification - from palace.manager.sqlalchemy.model.coverage import CoverageRecord from palace.manager.sqlalchemy.model.credential import Credential from palace.manager.sqlalchemy.model.customlist import CustomList from palace.manager.sqlalchemy.model.edition import Edition @@ -54,11 +53,6 @@ def active_name(self) -> str: "Edition", back_populates="data_source", uselist=True ) - # One DataSource can generate many CoverageRecords. - coverage_records: Mapped[list[CoverageRecord]] = relationship( - "CoverageRecord", back_populates="data_source" - ) - # One DataSource can generate many IDEquivalencies. id_equivalencies: Mapped[list[Equivalency]] = relationship( "Equivalency", back_populates="data_source" diff --git a/src/palace/manager/sqlalchemy/model/identifier.py b/src/palace/manager/sqlalchemy/model/identifier.py index 762377088b..130d9049a4 100644 --- a/src/palace/manager/sqlalchemy/model/identifier.py +++ b/src/palace/manager/sqlalchemy/model/identifier.py @@ -41,7 +41,6 @@ from palace.manager.sqlalchemy.constants import IdentifierConstants, LinkRelations from palace.manager.sqlalchemy.model.base import Base from palace.manager.sqlalchemy.model.classification import Classification, Subject -from palace.manager.sqlalchemy.model.coverage import CoverageRecord from palace.manager.sqlalchemy.model.datasource import DataSource from palace.manager.sqlalchemy.model.licensing import ( LicensePool, @@ -260,11 +259,6 @@ def active_type(self) -> str: uselist=True, ) - # One Identifier may have many associated CoverageRecords. - coverage_records: Mapped[list[CoverageRecord]] = relationship( - "CoverageRecord", back_populates="identifier" - ) - def __repr__(self) -> str: records = self.primarily_identifies if records and records[0].title: @@ -1175,50 +1169,6 @@ def evaluate_summary_quality( ) return champion, descriptions - @classmethod - def missing_coverage_from( - cls, - _db: Session, - identifier_types: list[str] | None, - coverage_data_source: DataSource | None, - operation: str | None = None, - count_as_covered: list[str] | None = None, - count_as_missing_before: datetime.datetime | None = None, - identifiers: list[Identifier] | None = None, - collection: Collection | None = None, - ) -> Query[Identifier]: - """Find identifiers of the given types which have no CoverageRecord - from `coverage_data_source`. - :param count_as_covered: Identifiers will be counted as - covered if their CoverageRecords have a status in this list. - :param identifiers: Restrict search to a specific set of identifier objects. - """ - if collection: - collection_id = collection.id - else: - collection_id = None - - data_source_id = None - if coverage_data_source: - data_source_id = coverage_data_source.id - - clause = and_( - Identifier.id == CoverageRecord.identifier_id, - CoverageRecord.data_source_id == data_source_id, - CoverageRecord.operation == operation, - CoverageRecord.collection_id == collection_id, - ) - qu = _db.query(Identifier).outerjoin(CoverageRecord, clause) - if identifier_types: - qu = qu.filter(Identifier.type.in_(identifier_types)) - missing = CoverageRecord.not_covered(count_as_covered, count_as_missing_before) - qu = qu.filter(missing) - - if identifiers: - qu = qu.filter(Identifier.id.in_([x.id for x in identifiers])) - - return qu - def __eq__(self, other: object) -> bool: """Equality implementation for total_ordering.""" # We don't want an Identifier to be == an IdentifierData diff --git a/tests/fixtures/celery.py b/tests/fixtures/celery.py index 985dd48ad0..c86a61a308 100644 --- a/tests/fixtures/celery.py +++ b/tests/fixtures/celery.py @@ -192,7 +192,6 @@ def mock_apply(self, db: Session) -> None: edition, collection, replace=self.replace, - create_coverage_record=False, ) diff --git a/tests/fixtures/database.py b/tests/fixtures/database.py index 8814e2c45f..4f693b4b52 100644 --- a/tests/fixtures/database.py +++ b/tests/fixtures/database.py @@ -86,7 +86,6 @@ ) from palace.manager.sqlalchemy.model.collection import Collection from palace.manager.sqlalchemy.model.contributor import Contributor -from palace.manager.sqlalchemy.model.coverage import CoverageRecord from palace.manager.sqlalchemy.model.credential import Credential from palace.manager.sqlalchemy.model.customlist import CustomList from palace.manager.sqlalchemy.model.datasource import DataSource @@ -1054,34 +1053,6 @@ def subject(self, type, identifier) -> Subject: self.session, Subject, type=type, identifier=identifier )[0] - def coverage_record( - self, - edition, - coverage_source, - operation=None, - status=CoverageRecord.SUCCESS, - collection=None, - exception=None, - ) -> CoverageRecord: - if isinstance(edition, Identifier): - identifier = edition - else: - identifier = edition.primary_identifier - record, ignore = get_one_or_create( - self.session, - CoverageRecord, - identifier=identifier, - data_source=coverage_source, - operation=operation, - collection=collection, - create_method_kwargs=dict( - timestamp=utc_now(), - status=status, - exception=exception, - ), - ) - return record - def identifier(self, identifier_type=Identifier.GUTENBERG_ID, foreign_id=None): if foreign_id: id_value = foreign_id diff --git a/tests/manager/api/admin/controller/test_work_editor.py b/tests/manager/api/admin/controller/test_work_editor.py index 612b35f73b..cc46704e0f 100644 --- a/tests/manager/api/admin/controller/test_work_editor.py +++ b/tests/manager/api/admin/controller/test_work_editor.py @@ -32,7 +32,6 @@ Subject, ) from palace.manager.sqlalchemy.model.contributor import Contributor -from palace.manager.sqlalchemy.model.coverage import CoverageRecord from palace.manager.sqlalchemy.model.customlist import CustomList from palace.manager.sqlalchemy.model.datasource import DataSource from palace.manager.sqlalchemy.model.edition import Edition @@ -43,10 +42,6 @@ from tests.fixtures.api_controller import ControllerFixture from tests.fixtures.database import DatabaseTransactionFixture from tests.fixtures.problem_detail import raises_problem_detail -from tests.mocks.mock import ( - AlwaysSuccessfulCoverageProvider, - NeverSuccessfulCoverageProvider, -) class WorkFixture(AdminControllerFixture): @@ -824,39 +819,11 @@ def test_unsuppress(self, work_fixture: WorkFixture): assert exc.value.problem_detail == LIBRARY_NOT_FOUND def test_refresh_metadata(self, work_fixture: WorkFixture): - wrangler = DataSource.lookup( - work_fixture.ctrl.db.session, DataSource.METADATA_WRANGLER - ) - - class AlwaysSuccessfulMetadataProvider(AlwaysSuccessfulCoverageProvider): - DATA_SOURCE_NAME = wrangler.name - - success_provider = AlwaysSuccessfulMetadataProvider( - work_fixture.ctrl.db.session - ) - - class NeverSuccessfulMetadataProvider(NeverSuccessfulCoverageProvider): - DATA_SOURCE_NAME = wrangler.name - - failure_provider = NeverSuccessfulMetadataProvider(work_fixture.ctrl.db.session) - + # Metadata refresh used to run through the per-source CoverageProvider + # machinery, which has been retired. The endpoint remains for API + # compatibility but now always reports failure. with work_fixture.request_context_with_library_and_admin("/"): [lp] = work_fixture.english_1.license_pools - response = work_fixture.manager.admin_work_controller.refresh_metadata( - lp.identifier.type, lp.identifier.identifier, provider=success_provider - ) - assert 200 == response.status_code - # Also, the work has a coverage record now for the wrangler. - assert CoverageRecord.lookup(lp.identifier, wrangler) - - response = work_fixture.manager.admin_work_controller.refresh_metadata( - lp.identifier.type, lp.identifier.identifier, provider=failure_provider - ) - assert METADATA_REFRESH_FAILURE.status_code == response.status_code - assert METADATA_REFRESH_FAILURE.detail == response.detail - - # If we don't pass in a provider, it will also fail because there - # isn't one connfigured. response = work_fixture.manager.admin_work_controller.refresh_metadata( lp.identifier.type, lp.identifier.identifier ) @@ -872,7 +839,6 @@ class NeverSuccessfulMetadataProvider(NeverSuccessfulCoverageProvider): work_fixture.manager.admin_work_controller.refresh_metadata, lp.identifier.type, lp.identifier.identifier, - provider=success_provider, ) def test_classifications(self, work_fixture: WorkFixture): diff --git a/tests/manager/core/test_coverage.py b/tests/manager/core/test_coverage.py deleted file mode 100644 index 6bc1f4e18d..0000000000 --- a/tests/manager/core/test_coverage.py +++ /dev/null @@ -1,1957 +0,0 @@ -import datetime -from typing import cast - -import pytest - -from palace.util.datetime_helpers import datetime_utc, utc_now - -from palace.manager.core.coverage import ( - BaseCoverageProvider, - CoverageFailure, - CoverageProviderProgress, - IdentifierCoverageProvider, -) -from palace.manager.data_layer.bibliographic import BibliographicData -from palace.manager.data_layer.circulation import CirculationData -from palace.manager.data_layer.contributor import ContributorData -from palace.manager.data_layer.format import FormatData -from palace.manager.data_layer.identifier import IdentifierData -from palace.manager.data_layer.link import LinkData -from palace.manager.data_layer.policy.presentation import ( - PresentationCalculationPolicy, -) -from palace.manager.data_layer.policy.replacement import ReplacementPolicy -from palace.manager.data_layer.subject import SubjectData -from palace.manager.integration.license.boundless.api import BoundlessApi -from palace.manager.integration.license.opds.opds1.api import OPDSAPI -from palace.manager.integration.license.overdrive.api import OverdriveAPI -from palace.manager.sqlalchemy.model.classification import Subject -from palace.manager.sqlalchemy.model.collection import CollectionMissing -from palace.manager.sqlalchemy.model.contributor import Contributor -from palace.manager.sqlalchemy.model.coverage import ( - CoverageRecord, - Timestamp, -) -from palace.manager.sqlalchemy.model.datasource import DataSource -from palace.manager.sqlalchemy.model.identifier import Identifier -from palace.manager.sqlalchemy.model.licensing import ( - DeliveryMechanism, - LicensePool, - RightsStatus, -) -from palace.manager.sqlalchemy.model.resource import Hyperlink, Representation -from palace.manager.sqlalchemy.model.work import Work -from tests.fixtures.database import DatabaseTransactionFixture -from tests.mocks.mock import ( - AlwaysSuccessfulBibliographicCoverageProvider, - AlwaysSuccessfulCollectionCoverageProvider, - AlwaysSuccessfulCoverageProvider, - NeverSuccessfulBibliographicCoverageProvider, - NeverSuccessfulCoverageProvider, - TaskIgnoringCoverageProvider, - TransientFailureCoverageProvider, -) - - -class TestCoverageProviderProgress: - def test_achievements(self): - progress = CoverageProviderProgress() - progress.successes = 1 - progress.transient_failures = 2 - progress.persistent_failures = 0 - - expect = "Items processed: 3. Successes: 1, transient failures: 2, persistent failures: 0" - assert expect == progress.achievements - - # You can't set .achievements directly -- it's a calculated value. - progress.achievements = "new value" - assert expect == progress.achievements - - -@pytest.fixture -def bibliographic_data() -> BibliographicData: - return BibliographicData( - data_source_name=DataSource.OVERDRIVE, - publisher="Perfection Learning", - language="eng", - title="A Girl Named Disaster", - published=datetime_utc(1998, 3, 1, 0, 0), - primary_identifier_data=IdentifierData( - type=Identifier.OVERDRIVE_ID, - identifier="ba9b3419-b0bd-4ca7-a24f-26c4246b6b44", - ), - identifiers=[ - IdentifierData( - type=Identifier.OVERDRIVE_ID, - identifier="ba9b3419-b0bd-4ca7-a24f-26c4246b6b44", - ), - IdentifierData(type=Identifier.ISBN, identifier="9781402550805"), - ], - contributors=[ - ContributorData( - sort_name="Nancy Farmer", roles=[Contributor.Role.PRIMARY_AUTHOR] - ) - ], - subjects=[ - SubjectData(type=Subject.TOPIC, identifier="Action & Adventure"), - SubjectData(type=Subject.FREEFORM_AUDIENCE, identifier="Young Adult"), - SubjectData(type=Subject.PLACE, identifier="Africa"), - ], - ) - - -@pytest.fixture -def circulation_data(bibliographic_data: BibliographicData) -> CirculationData: - # This data is used to test the insertion of circulation data - # into a Collection. - return CirculationData( - data_source_name=DataSource.OVERDRIVE, - primary_identifier_data=bibliographic_data.primary_identifier_data, - formats=[ - FormatData( - content_type=Representation.EPUB_MEDIA_TYPE, - drm_scheme=DeliveryMechanism.NO_DRM, - rights_uri=RightsStatus.IN_COPYRIGHT, - ) - ], - ) - - -class TestBaseCoverageProvider: - def test_instantiation(self, db: DatabaseTransactionFixture): - """Verify variable initialization.""" - - class ValidMock(BaseCoverageProvider): - SERVICE_NAME = "A Service" - OPERATION = "An Operation" - DEFAULT_BATCH_SIZE = 50 - - now = cutoff_time = utc_now() - provider = ValidMock(db.session, cutoff_time=now) - - # Class variables defined in subclasses become appropriate - # instance variables. - assert "A Service (An Operation)" == provider.service_name - assert "An Operation" == provider.operation - assert 50 == provider.batch_size - assert now == provider.cutoff_time - - # If you pass in an invalid value for batch_size, you get the default. - provider = ValidMock(db.session, batch_size=-10) - assert 50 == provider.batch_size - - def test_subclass_must_define_service_name(self, db: DatabaseTransactionFixture): - class NoServiceName(BaseCoverageProvider): - pass - - with pytest.raises(ValueError) as excinfo: - NoServiceName(db.session) - assert "NoServiceName must define SERVICE_NAME" in str(excinfo.value) - - def test_run(self, db: DatabaseTransactionFixture): - """Verify that run() calls run_once_and_update_timestamp().""" - - class MockProvider(BaseCoverageProvider): - SERVICE_NAME = "I do nothing" - was_run = False - - def run_once_and_update_timestamp(self): - """Set a variable.""" - self.was_run = True - return None - - provider = MockProvider(db.session) - result = provider.run() - - # run_once_and_update_timestamp() was called. - assert True == provider.was_run - - # run() returned a CoverageProviderProgress with basic - # timing information, since run_once_and_update_timestamp() - # didn't provide anything. - assert isinstance(result, CoverageProviderProgress) - now = utc_now() - assert isinstance(result.start, datetime.datetime) - assert isinstance(result.finish, datetime.datetime) - assert result.start < result.finish - for time in (result.start, result.finish): - assert (now - time).total_seconds() < 5 - - def test_run_with_custom_result(self, db: DatabaseTransactionFixture): - start = datetime_utc(2011, 1, 1) - finish = datetime_utc(2012, 1, 1) - counter = -100 - - class MockProvider(BaseCoverageProvider): - """A BaseCoverageProvider that returns a strange - CoverageProviderProgress representing the work it did. - """ - - SERVICE_NAME = "I do nothing" - was_run = False - - custom_timestamp_data = CoverageProviderProgress( - start=start, finish=finish, counter=counter - ) - - def run_once_and_update_timestamp(self): - return self.custom_timestamp_data - - provider = MockProvider(db.session) - result = provider.run() - - # The TimestampData returned by run_once_and_update_timestamp - # is the return value of run(). - assert result == provider.custom_timestamp_data - - # The TimestampData data was written to the database, even - # though some of it doesn't make apparent sense. - assert start == provider.timestamp.start - assert finish == provider.timestamp.finish - assert counter == provider.timestamp.counter - - def test_run_once_and_update_timestamp(self, db: DatabaseTransactionFixture): - """Test that run_once_and_update_timestamp calls run_once until all - the work is done, and then updates a Timestamp. - """ - - class MockProvider(BaseCoverageProvider): - SERVICE_NAME = "I do nothing" - run_once_calls = [] - expect_offset = 0 - - def run_once(self, progress, count_as_covered=None): - now = utc_now() - - # We never see progress.finish set to a non-None - # value. When _we_ set it to a non-None value, it means - # the work is done. If we get called again, it'll be - # with different `count_as_covered` settings, and - # .finish will have been reset to None. - assert None == progress.finish - - # Verify that progress.offset is cleared when we - # expect, and left alone when we expect. This lets - assert self.expect_offset == progress.offset - - self.run_once_calls.append((count_as_covered, now)) - progress.offset = len(self.run_once_calls) - - if len(self.run_once_calls) == 1: - # This is the first call. We will not be setting - # .finish, so the offset will not be reset on the - # next call. This simulates what happens when a - # given `count_as_covered` setting can't be - # handled in one batch. - self.expect_offset = progress.offset - else: - # This is the second or third call. Set .finish to - # indicate we're done with this `count_as_covered` - # setting. - progress.finish = now - - # If there is another call, progress.offset will be - # reset to zero. (So will .finish.) - self.expect_offset = 0 - return progress - - # We start with no Timestamp. - service_name = "I do nothing" - service_type = Timestamp.COVERAGE_PROVIDER_TYPE - timestamp = Timestamp.value( - db.session, service_name, service_type, collection=None - ) - assert None == timestamp - - # Instantiate the Provider, and call - # run_once_and_update_timestamp. - provider = MockProvider(db.session) - final_progress = provider.run_once_and_update_timestamp() - - # The Timestamp's .start and .finish are now set to recent - # values -- the start and end points of run_once(). - timestamp = provider.timestamp - now = utc_now() - assert (now - timestamp.start).total_seconds() < 1 - assert (now - timestamp.finish).total_seconds() < 1 - assert timestamp.start < timestamp.finish - - # run_once was called three times: twice to exclude items that - # have any coverage record whatsoever (PREVIOUSLY_ATTEMPTED), - # and a third time to exclude only items that have coverage - # records that indicate success or persistent failure - # (DEFAULT_COUNT_AS_COVERED). - first_call, second_call, third_call = provider.run_once_calls - assert CoverageRecord.PREVIOUSLY_ATTEMPTED == first_call[0] - assert CoverageRecord.PREVIOUSLY_ATTEMPTED == second_call[0] - assert CoverageRecord.DEFAULT_COUNT_AS_COVERED == third_call[0] - - # On the second and third calls, final_progress.finish was set - # to the current time, and .offset was set to the number of - # calls so far. - # - # These values are cleared out before each run_once() call - # -- we tested that above -- so the surviving values are the - # ones associated with the third call. - assert third_call[1] == final_progress.finish - assert 3 == final_progress.offset - - def test_run_once_and_update_timestamp_catches_exception( - self, db: DatabaseTransactionFixture - ): - # Test that run_once_and_update_timestamp catches an exception - # and stores a stack trace in the CoverageProvider's Timestamp. - class MockProvider(BaseCoverageProvider): - SERVICE_NAME = "I fail" - - def run_once(self, progress, count_as_covered=None): - raise Exception("Unhandled exception") - - provider = MockProvider(db.session) - provider.run_once_and_update_timestamp() - - timestamp = provider.timestamp - now = utc_now() - assert (now - timestamp.start).total_seconds() < 1 - assert (now - timestamp.finish).total_seconds() < 1 - assert timestamp.start < timestamp.finish - - assert "Exception: Unhandled exception" in timestamp.exception - - def test_run_once_and_update_timestamp_handled_exception( - self, db: DatabaseTransactionFixture - ): - # Test that run_once_and_update_timestamp handles the - # case where the run_once() implementation sets TimestampData.exception - # rather than raising an exception. - # - # This also tests the case where run_once() modifies the - # TimestampData in place rather than returning a new one. - class MockProvider(BaseCoverageProvider): - SERVICE_NAME = "I fail" - - def run_once(self, progress, count_as_covered=None): - progress.exception = "oops" - - provider = MockProvider(db.session) - provider.run_once_and_update_timestamp() - - timestamp = provider.timestamp - now = utc_now() - assert (now - timestamp.start).total_seconds() < 1 - assert (now - timestamp.finish).total_seconds() < 1 - assert timestamp.start < timestamp.finish - - assert "oops" == timestamp.exception - - def test_run_once(self, db: DatabaseTransactionFixture): - # Test run_once, showing how it covers items with different types of - # CoverageRecord. - - # We start with no CoverageRecords. - assert [] == db.session.query(CoverageRecord).all() - - # Four identifiers. - transient = db.identifier() - persistent = db.identifier() - uncovered = db.identifier() - covered = db.identifier() - - # This provider will try to cover them. - provider = AlwaysSuccessfulCoverageProvider(db.session) - data_source = provider.data_source - - # We previously tried to cover one of them, but got a - # transient failure. - db.coverage_record( - transient, data_source, status=CoverageRecord.TRANSIENT_FAILURE - ) - - # Another of the four has a persistent failure. - db.coverage_record( - persistent, data_source, status=CoverageRecord.PERSISTENT_FAILURE - ) - - # The third one has no coverage record at all. - - # And the fourth one has been successfully covered. - db.coverage_record(covered, data_source, status=CoverageRecord.SUCCESS) - - # Now let's run the coverage provider. Every Identifier - # that's covered will succeed, so the question is which ones - # get covered. - progress = CoverageProviderProgress() - assert 0 == progress.offset - result = provider.run_once(progress) - - # The TimestampData we passed in was given back to us. - assert progress == result - - # The offset (an extension specific to - # CoverageProviderProgress, not stored in the database) - # has not changed -- if we were to call run_once again we - # would not need to skip any records. - assert 0 == progress.offset - - # Various internal totals were updated and a value for .achievements - # can be generated from those totals. - assert 2 == progress.successes - - # By default, run_once() finds Identifiers that have no coverage - # or which have transient failures. - [transient_failure_has_gone] = transient.coverage_records - assert CoverageRecord.SUCCESS == transient_failure_has_gone.status - - [now_has_coverage] = uncovered.coverage_records - assert CoverageRecord.SUCCESS == now_has_coverage.status - - assert transient in provider.attempts - assert uncovered in provider.attempts - - # Nothing happened to the identifier that had a persistent - # failure or the identifier that was successfully covered. - assert [CoverageRecord.PERSISTENT_FAILURE] == [ - x.status for x in persistent.coverage_records - ] - assert [CoverageRecord.SUCCESS] == [x.status for x in covered.coverage_records] - - assert persistent not in provider.attempts - assert covered not in provider.attempts - - # We can change which identifiers get processed by changing - # what counts as 'coverage'. - result = provider.run_once(progress, count_as_covered=[CoverageRecord.SUCCESS]) - assert progress == result - assert 0 == progress.offset - - # That processed the persistent failure, but not the success. - assert persistent in provider.attempts - assert covered not in provider.attempts - - # Let's call it again and say that we are covering everything - # _except_ persistent failures. - result = provider.run_once( - progress, count_as_covered=[CoverageRecord.PERSISTENT_FAILURE] - ) - assert progress == result - - # That got us to cover the identifier that had already been - # successfully covered. - assert covered in provider.attempts - - # *Now* the offset has changed, so that the first four results - # -- which we've decided to skip -- won't be considered again - # this run. - assert 4 == progress.offset - - def test_run_once_records_successes_and_failures( - self, db: DatabaseTransactionFixture - ): - class Mock(AlwaysSuccessfulCoverageProvider): - def process_batch_and_handle_results(self, batch): - # Simulate 1 success, 2 transient failures, - # and 3 persistent failures. - return (1, 2, 3), [] - - # process_batch_and_handle_results won't even be called if the - # batch is empty. - provider = Mock(db.session) - progress = CoverageProviderProgress() - progress2 = provider.run_once(progress) - assert progress2 == progress - assert 0 == progress.successes - - # Let's register an identifier so that the method we're testing - # will be called. - needs_coverage = db.identifier() - progress = provider.run_once(progress) - - # The numbers returned from process_batch_and_handle_results - # were added to the CoverageProviderProgress object. - assert 1 == progress.successes - assert 2 == progress.transient_failures - assert 3 == progress.persistent_failures - assert provider.finalize_batch_called - - assert ( - "Items processed: 6. Successes: 1, transient failures: 2, persistent failures: 3" - == progress.achievements - ) - - def test_process_batch_and_handle_results(self, db: DatabaseTransactionFixture): - """Test that process_batch_and_handle_results passes the identifiers - its given into the appropriate BaseCoverageProvider, and deals - correctly with the successes and failures it might return. - """ - e1, p1 = db.edition(with_license_pool=True) - i1 = e1.primary_identifier - - e2, p2 = db.edition(with_license_pool=True) - i2 = e2.primary_identifier - - class MockProvider1(AlwaysSuccessfulCoverageProvider): - OPERATION = "i succeed" - - success_provider = MockProvider1(db.session) - - batch = [i1, i2] - counts, successes = success_provider.process_batch_and_handle_results(batch) - - # Two successes. - assert (2, 0, 0) == counts - - # Each represented with a CoverageRecord with status='success' - assert all(isinstance(x, CoverageRecord) for x in successes) - assert [CoverageRecord.SUCCESS] * 2 == [x.status for x in successes] - - # Each associated with one of the identifiers... - assert {i1, i2} == {x.identifier for x in successes} - - # ...and with the coverage provider's operation. - assert ["i succeed"] * 2 == [x.operation for x in successes] - - # Now try a different CoverageProvider which creates transient - # failures. - class MockProvider2(TransientFailureCoverageProvider): - OPERATION = "i fail transiently" - - transient_failure_provider = MockProvider2(db.session) - counts, failures = transient_failure_provider.process_batch_and_handle_results( - batch - ) - # Two transient failures. - assert (0, 2, 0) == counts - - # New coverage records were added to track the transient - # failures. - assert [CoverageRecord.TRANSIENT_FAILURE] * 2 == [x.status for x in failures] - assert ["i fail transiently"] * 2 == [x.operation for x in failures] - - # Another way of getting transient failures is to just ignore every - # item you're told to process. - class MockProvider3(TaskIgnoringCoverageProvider): - OPERATION = "i ignore" - - task_ignoring_provider = MockProvider3(db.session) - counts, records = task_ignoring_provider.process_batch_and_handle_results(batch) - - assert (0, 2, 0) == counts - assert [CoverageRecord.TRANSIENT_FAILURE] * 2 == [x.status for x in records] - assert ["i ignore"] * 2 == [x.operation for x in records] - - # If a transient failure becomes a success, the it won't have - # an exception anymore. - assert ["Was ignored by CoverageProvider."] * 2 == [ - x.exception for x in records - ] - records = success_provider.process_batch_and_handle_results(batch)[1] - assert [None, None] == [x.exception for x in records] - - # Or you can go really bad and have persistent failures. - class MockProvider4(NeverSuccessfulCoverageProvider): - OPERATION = "i will always fail" - - persistent_failure_provider = MockProvider4(db.session) - counts, results = persistent_failure_provider.process_batch_and_handle_results( - batch - ) - - # Two persistent failures. - assert (0, 0, 2) == counts - assert all([isinstance(x, CoverageRecord) for x in results]) - assert ["What did you expect?", "What did you expect?"] == [ - x.exception for x in results - ] - assert [CoverageRecord.PERSISTENT_FAILURE] * 2 == [x.status for x in results] - assert ["i will always fail"] * 2 == [x.operation for x in results] - - assert not success_provider.finalize_batch_called - assert not persistent_failure_provider.finalize_batch_called - - def test_process_batch(self, db: DatabaseTransactionFixture): - class Mock(BaseCoverageProvider): - SERVICE_NAME = "Some succeed, some fail." - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.processed = [] - self.successes = [] - - def process_item(self, item): - self.processed.append(item) - if item.identifier == "fail": - return CoverageFailure(item, "oops") - return item - - def handle_success(self, item): - self.successes.append(item) - - # Two Identifiers. One will succeed, one will fail. - succeed = db.identifier(foreign_id="succeed") - fail = db.identifier(foreign_id="fail") - provider = Mock(db.session) - - r1, r2 = provider.process_batch([succeed, fail]) - - # Here's the success. - assert r1 == succeed - - # Here's the failure. - assert isinstance(r2, CoverageFailure) - assert "oops" == r2.exception - - # Both identifiers were added to .processed, indicating that - # process_item was called twice, but only the success was - # added to .success, indicating that handle_success was only - # called once. - assert [succeed, fail] == provider.processed - assert [succeed] == provider.successes - - def test_should_update(self, db: DatabaseTransactionFixture): - """Verify that should_update gives the correct answer when we - ask if a CoverageRecord needs to be updated. - """ - cutoff = datetime_utc(2016, 1, 1) - provider = AlwaysSuccessfulCoverageProvider(db.session, cutoff_time=cutoff) - identifier = db.identifier() - - # If coverage is missing, we should update. - assert True == provider.should_update(None) - - # If coverage is outdated, we should update. - record, ignore = CoverageRecord.add_for(identifier, provider.data_source) - record.timestamp = datetime_utc(2015, 1, 1) - assert True == provider.should_update(record) - - # If coverage is up-to-date, we should not update. - record.timestamp = cutoff - assert False == provider.should_update(record) - - # If coverage is only 'registered', we should update. - record.status = CoverageRecord.REGISTERED - assert True == provider.should_update(record) - - -class TestIdentifierCoverageProvider: - def test_input_identifier_types(self, db: DatabaseTransactionFixture): - """Test various acceptable and unacceptable values for the class - variable INPUT_IDENTIFIER_TYPES. - """ - - # It's okay to set INPUT_IDENTIFIER_TYPES to None it means you - # will cover any and all identifier types. - class Base(IdentifierCoverageProvider): - SERVICE_NAME = "Test provider" - DATA_SOURCE_NAME = DataSource.GUTENBERG - - class MockProvider1(Base): - INPUT_IDENTIFIER_TYPES = None - - provider = MockProvider1(db.session) - assert None == provider.input_identifier_types - - # It's okay to set a single value. - class MockProvider2(Base): - INPUT_IDENTIFIER_TYPES = Identifier.ISBN - - provider2 = MockProvider2(db.session) - assert [Identifier.ISBN] == provider2.input_identifier_types - - # It's okay to set a list of values. - class MockProvider3(Base): - INPUT_IDENTIFIER_TYPES = [Identifier.ISBN, Identifier.OVERDRIVE_ID] - - provider3 = MockProvider3(db.session) - assert [ - Identifier.ISBN, - Identifier.OVERDRIVE_ID, - ] == provider3.input_identifier_types - - # It's not okay to do nothing. - class MockProvider4(Base): - pass - - with pytest.raises(ValueError) as excinfo: - MockProvider4(db.session) - assert ( - "MockProvider4 must define INPUT_IDENTIFIER_TYPES, even if the value is None." - in str(excinfo.value) - ) - - def test_can_cover(self, db: DatabaseTransactionFixture): - """Verify that can_cover gives the correct answer when - asked if an IdentifierCoverageProvider can handle a given Identifier. - """ - provider = AlwaysSuccessfulCoverageProvider(db.session) - identifier = db.identifier(identifier_type=Identifier.ISBN) - m = provider.can_cover - - # This provider handles all identifier types. - provider.input_identifier_types = None - assert True == m(identifier) - - # This provider handles ISBNs. - provider.input_identifier_types = [Identifier.OVERDRIVE_ID, Identifier.ISBN] - assert True == m(identifier) - - # This provider doesn't. - provider.input_identifier_types = [Identifier.OVERDRIVE_ID] - assert False == m(identifier) - - def test_replacement_policy(self, db: DatabaseTransactionFixture): - """Unless a different replacement policy is passed in, the - default is ReplacementPolicy.from_metadata_source(). - """ - provider = AlwaysSuccessfulCoverageProvider(db.session) - assert True == provider.replacement_policy.identifiers - assert False == provider.replacement_policy.formats - - policy = ReplacementPolicy.from_license_source() - provider = AlwaysSuccessfulCoverageProvider( - db.session, replacement_policy=policy - ) - assert policy == provider.replacement_policy - - def test_register(self, db: DatabaseTransactionFixture): - # The identifier has no coverage. - identifier = db.identifier() - assert 0 == len(identifier.coverage_records) - - provider = AlwaysSuccessfulCoverageProvider - - # If a CoverageRecord doesn't exist for the provider, - # a 'registered' record is created. - new_record, was_registered = provider.register(identifier) - - assert identifier.coverage_records == [new_record] - assert provider.DATA_SOURCE_NAME == new_record.data_source.name - assert CoverageRecord.REGISTERED == new_record.status - assert None == new_record.exception - - # If a CoverageRecord exists already, it's returned. - existing = new_record - existing.status = CoverageRecord.SUCCESS - - new_record, was_registered = provider.register(identifier) - assert existing == new_record - assert False == was_registered - # Its details haven't been changed in any way. - assert CoverageRecord.SUCCESS == new_record.status - assert None == new_record.exception - - def test_bulk_register(self, db: DatabaseTransactionFixture): - provider = AlwaysSuccessfulCoverageProvider - source = DataSource.lookup(db.session, provider.DATA_SOURCE_NAME) - - i1 = db.identifier() - covered = db.identifier() - existing = db.coverage_record(covered, source, operation=provider.OPERATION) - - new_records, ignored_identifiers = provider.bulk_register([i1, covered]) - - assert i1.coverage_records == new_records - [new_record] = new_records - assert provider.DATA_SOURCE_NAME == new_record.data_source.name - assert provider.OPERATION == new_record.operation - assert CoverageRecord.REGISTERED == new_record.status - - assert [covered] == ignored_identifiers - # The existing CoverageRecord hasn't been changed. - assert CoverageRecord.SUCCESS == existing.status - - def test_bulk_register_can_overwrite_existing_record_status( - self, db: DatabaseTransactionFixture - ): - provider = AlwaysSuccessfulCoverageProvider - identifier = db.identifier() - - # Create an existing record, and give it a SUCCESS status. - provider.bulk_register([identifier]) - [existing] = identifier.coverage_records - existing.status = CoverageRecord.SUCCESS - db.session.commit() - - # If registration is forced, an existing record is updated. - records, ignored = provider.bulk_register([identifier], force=True) - assert [existing] == records - assert CoverageRecord.REGISTERED == existing.status - - def test_bulk_register_with_collection(self, db: DatabaseTransactionFixture): - identifier = db.identifier() - provider = AlwaysSuccessfulCoverageProvider - collection = db.collection(protocol=BoundlessApi) - - try: - # If a DataSource or data source name is provided and - # autocreate is set True, the record is created with that source. - provider.bulk_register( - [identifier], - data_source=collection.name, - collection=collection, - autocreate=True, - ) - [record] = identifier.coverage_records - - # A DataSource with the given name has been created. - collection_source = DataSource.lookup(db.session, collection.name) - assert collection_source - assert provider.DATA_SOURCE_NAME != record.data_source.name - assert collection_source == record.data_source - - # Even though a collection was given, the record's collection isn't - # set. - assert None == record.collection - - # However, when coverage is collection-specific the - # CoverageRecord is related to the given collection. - provider.COVERAGE_COUNTS_FOR_EVERY_COLLECTION = False - - provider.bulk_register( - [identifier], collection_source, collection=collection - ) - records = identifier.coverage_records - assert 2 == len(records) - assert [r for r in records if r.collection == collection] - finally: - # Return the mock class to its original state for other tests. - provider.COVERAGE_COUNTS_FOR_EVERY_COLLECTION = True - - def test_ensure_coverage(self, db: DatabaseTransactionFixture): - """Verify that ensure_coverage creates a CoverageRecord for an - Identifier, assuming that the CoverageProvider succeeds. - """ - identifier = db.identifier() - provider = AlwaysSuccessfulCollectionCoverageProvider(db.default_collection()) - provider.OPERATION = db.fresh_str() - record = provider.ensure_coverage(identifier) - assert isinstance(record, CoverageRecord) - assert identifier == record.identifier - assert provider.data_source == record.data_source - assert provider.OPERATION == record.operation - assert None == record.exception - - # There is now one CoverageRecord -- the one returned by - # ensure_coverage(). - [record2] = db.session.query(CoverageRecord).all() - assert record2 == record - - # Because this provider counts coverage in one Collection as - # coverage for all Collections, the coverage record was not - # associated with any particular collection. - assert True == provider.COVERAGE_COUNTS_FOR_EVERY_COLLECTION - assert None == record2.collection - - # The coverage provider's timestamp was not updated, because - # we're using ensure_coverage on a single record. - assert None == Timestamp.value( - db.session, - provider.service_name, - Timestamp.COVERAGE_PROVIDER_TYPE, - collection=None, - ) - - # Now let's try a CollectionCoverageProvider that needs to - # grant coverage separately for every collection. - provider.COVERAGE_COUNTS_FOR_EVERY_COLLECTION = False - record3 = provider.ensure_coverage(identifier) - - # This creates a new CoverageRecord associated with the - # provider's collection. - assert record3 != record2 - assert provider.collection == record3.collection - - def test_ensure_coverage_works_on_edition(self, db: DatabaseTransactionFixture): - """Verify that ensure_coverage() works on an Edition by covering - its primary identifier. - """ - edition = db.edition() - provider = AlwaysSuccessfulCoverageProvider(db.session) - record = provider.ensure_coverage(edition) - assert isinstance(record, CoverageRecord) - assert edition.primary_identifier == record.identifier - - def test_ensure_coverage_respects_operation(self, db: DatabaseTransactionFixture): - # Two providers with the same output source but different operations. - class Mock1(AlwaysSuccessfulCoverageProvider): - OPERATION = "foo" - - provider1 = Mock1(db.session) - - class Mock2(NeverSuccessfulCoverageProvider): - OPERATION = "bar" - - provider2 = Mock2(db.session) - - identifier = db.identifier() - # Ensure coverage from both providers. - coverage1 = provider1.ensure_coverage(identifier) - assert "foo" == coverage1.operation - old_timestamp = coverage1.timestamp - - coverage2 = provider2.ensure_coverage(identifier) - assert "bar" == coverage2.operation - - # There are now two CoverageRecords, one for each operation. - assert {coverage1, coverage2} == set(db.session.query(CoverageRecord)) - - # If we try to ensure coverage again, no work is done and we - # get the old coverage record back. - new_coverage = provider1.ensure_coverage(identifier) - assert new_coverage == coverage1 - new_coverage.timestamp = old_timestamp - - def test_ensure_coverage_persistent_coverage_failure( - self, db: DatabaseTransactionFixture - ): - identifier = db.identifier() - provider = NeverSuccessfulCoverageProvider(db.session) - failure = provider.ensure_coverage(identifier) - - # A CoverageRecord has been created to memorialize the - # persistent failure. - assert isinstance(failure, CoverageRecord) - assert "What did you expect?" == failure.exception - - # Here it is in the database. - [record] = db.session.query(CoverageRecord).all() - assert record == failure - - # The coverage provider's timestamp was not updated, because - # we're using ensure_coverage. - # The coverage provider's timestamp was not updated, because - # we're using ensure_coverage on a single record. - assert None == Timestamp.value( - db.session, - provider.service_name, - service_type=Timestamp.COVERAGE_PROVIDER_TYPE, - collection=None, - ) - - def test_ensure_coverage_transient_coverage_failure( - self, db: DatabaseTransactionFixture - ): - identifier = db.identifier() - provider = TransientFailureCoverageProvider(db.session) - failure = provider.ensure_coverage(identifier) - assert [failure] == identifier.coverage_records - assert CoverageRecord.TRANSIENT_FAILURE == failure.status - assert "Oops!" == failure.exception - - # Timestamp was not updated. - assert None == Timestamp.value( - db.session, - provider.service_name, - service_type=Timestamp.COVERAGE_PROVIDER_TYPE, - collection=None, - ) - - def test_ensure_coverage_changes_status(self, db: DatabaseTransactionFixture): - """Verify that processing an item that has a preexisting - CoverageRecord can change the status of that CoverageRecord. - """ - - identifier = db.identifier() - always = AlwaysSuccessfulCoverageProvider(db.session) - persistent = NeverSuccessfulCoverageProvider(db.session) - transient = TransientFailureCoverageProvider(db.session) - - # Cover the same identifier multiple times, simulating all - # possible states of a CoverageRecord. The same CoverageRecord - # is used every time and the status is changed appropriately - # after every run. - c1 = persistent.ensure_coverage(identifier, force=True) - assert CoverageRecord.PERSISTENT_FAILURE == c1.status - - c2 = transient.ensure_coverage(identifier, force=True) - assert c2 == c1 - assert CoverageRecord.TRANSIENT_FAILURE == c1.status - - c3 = always.ensure_coverage(identifier, force=True) - assert c3 == c1 - assert CoverageRecord.SUCCESS == c1.status - - c4 = persistent.ensure_coverage(identifier, force=True) - assert c4 == c1 - assert CoverageRecord.PERSISTENT_FAILURE == c1.status - - def test_edition(self, db: DatabaseTransactionFixture): - """Verify that CoverageProvider.edition() returns an appropriate - Edition, even when there is no associated Collection. - """ - # This CoverageProvider fetches bibliographic information - # from Overdrive. It is not capable of creating LicensePools - # because it has no Collection. - provider = AlwaysSuccessfulCoverageProvider(db.session) - assert None == provider.collection - - # Here's an Identifier, with no Editions. - identifier = db.identifier(identifier_type=Identifier.OVERDRIVE_ID) - assert [] == identifier.primarily_identifies - - # Calling CoverageProvider.edition() on the Identifier gives - # us a container for the provider's bibliographic information, - # as given to us by the provider's data source. - # - # It doesn't matter that there's no Collection, because the - # book's bibliographic information is the same across - # Collections. - edition = provider.edition(identifier) - assert provider.data_source == edition.data_source - assert [edition] == identifier.primarily_identifies - - # Calling edition() again gives us the same Edition as before. - edition2 = provider.edition(identifier) - assert edition == edition2 - - def test_set_bibliographic( - self, bibliographic_data, db: DatabaseTransactionFixture - ): - """Test that set_bibliographic can create and populate an - appropriate Edition. - - set_bibliographic is tested in more detail in - TestCollectionCoverageProvider. - """ - # Here's a provider that is not associated with any particular - # Collection. - provider = AlwaysSuccessfulCoverageProvider(db.session) - assert None == provider.collection - - # It can't set circulation data, because it's not a - # CollectionCoverageProvider. - assert not hasattr(provider, "set_bibliographic_and_circulation_data") - - # But it can set bibliographic data. - identifier = db.identifier( - identifier_type=Identifier.OVERDRIVE_ID, - foreign_id=bibliographic_data.primary_identifier_data.identifier, - ) - assert [] == identifier.primarily_identifies - result = provider.set_bibliographic(identifier, bibliographic_data) - - # Here's the proof. - edition = provider.edition(identifier) - assert "A Girl Named Disaster" == edition.title - - # If no bibliographic data is passed in, a CoverageFailure results. - result = provider.set_bibliographic(identifier, None) - assert isinstance(result, CoverageFailure) - assert ( - "Did not receive bibliographic data from input source" == result.exception - ) - - # If there's an exception setting the bibliographic data, a - # CoverageFailure results. This call raises a ValueError - # because the primary identifier & the edition's primary - # identifier don't match. - bibliographic_data.primary_identifier_data = IdentifierData( - type=Identifier.OVERDRIVE_ID, identifier="abcd" - ) - result = provider.set_bibliographic(identifier, bibliographic_data) - assert isinstance(result, CoverageFailure) - assert "ValueError" in result.exception - - def test_items_that_need_coverage_respects_registration_reqs( - self, db: DatabaseTransactionFixture - ): - identifier = db.identifier() - provider = AlwaysSuccessfulCoverageProvider(db.session, registered_only=True) - - items = provider.items_that_need_coverage() - assert identifier not in items - - # Once the identifier is registered, it shows up. - provider.register(identifier) - assert identifier in items - - # With a failing CoverageRecord, the item shows up. - [record] = identifier.coverage_records - record.status = CoverageRecord.TRANSIENT_FAILURE - record.exception = "Oh no!" - assert identifier in items - - def test_items_that_need_coverage_respects_operation( - self, db: DatabaseTransactionFixture - ): - # Here's a provider that carries out the 'foo' operation. - class Mock1(AlwaysSuccessfulCoverageProvider): - OPERATION = "foo" - - identifier = db.identifier() - provider = Mock1(db.session) - - # Here's a generic CoverageRecord for an identifier. - record1 = CoverageRecord.add_for(identifier, provider.data_source) - - # That record doesn't count for purposes of - # items_that_need_coverage, because the CoverageRecord doesn't - # have an operation, and the CoverageProvider does. - assert [identifier] == provider.items_that_need_coverage().all() - - # Here's a provider that has no operation set. - successful_provider = AlwaysSuccessfulCoverageProvider(db.session) - assert None == successful_provider.OPERATION - - # For purposes of items_that_need_coverage, the identifier is - # considered covered, because the operations match. - assert [] == successful_provider.items_that_need_coverage().all() - - def test_run_on_specific_identifiers(self, db: DatabaseTransactionFixture): - provider = AlwaysSuccessfulCoverageProvider(db.session) - to_be_tested = [db.identifier() for i in range(6)] - not_to_be_tested = [db.identifier() for i in range(6)] - counts, records = provider.run_on_specific_identifiers(to_be_tested) - - # Six identifiers were covered in two batches. - assert (6, 0, 0) == counts - assert 6 == len(records) - - # Only the identifiers in to_be_tested were covered. - assert all(isinstance(x, CoverageRecord) for x in records) - assert set(to_be_tested) == {x.identifier for x in records} - for i in to_be_tested: - assert i in provider.attempts - for i in not_to_be_tested: - assert i not in provider.attempts - - assert provider.finalize_batch_called - - def test_run_on_specific_identifiers_respects_cutoff_time( - self, db: DatabaseTransactionFixture - ): - last_run = datetime_utc(2016, 1, 1) - - # Once upon a time we successfully added coverage for - # self.identifier. But now something has gone wrong, and if we - # ever run the coverage provider again we will get a - # persistent failure. - identifier = db.identifier() - provider = NeverSuccessfulCoverageProvider(db.session) - record, ignore = CoverageRecord.add_for(identifier, provider.data_source) - record.timestamp = last_run - - # You might think this would result in a persistent failure... - ( - success, - transient_failure, - persistent_failure, - ), records = provider.run_on_specific_identifiers([identifier]) - - # ...but we get an automatic success. We didn't even try to - # run the coverage provider on self.identifier because the - # coverage record was up-to-date. - assert 1 == success - assert 0 == persistent_failure - assert [] == records - - # But if we move the cutoff time forward, the provider will run - # on self.identifier and fail. - provider.cutoff_time = datetime_utc(2016, 2, 1) - ( - success, - transient_failure, - persistent_failure, - ), records = provider.run_on_specific_identifiers([identifier]) - assert 0 == success - assert 1 == persistent_failure - - # The formerly successful CoverageRecord will be updated to - # reflect the failure. - assert records[0] == record - assert "What did you expect?" == record.exception - assert provider.finalize_batch_called - - def test_run_never_successful(self, db: DatabaseTransactionFixture): - """Verify that NeverSuccessfulCoverageProvider works the - way we'd expect. - """ - - identifier = db.identifier() - provider = NeverSuccessfulCoverageProvider(db.session) - - # We start with no CoverageRecords and no Timestamp. - assert [] == db.session.query(CoverageRecord).all() - assert None == Timestamp.value( - db.session, - provider.service_name, - service_type=Timestamp.COVERAGE_PROVIDER_TYPE, - collection=None, - ) - - provider.run() - - # We have a CoverageRecord that signifies failure. - [record] = db.session.query(CoverageRecord).all() - assert identifier == record.identifier - assert record.data_source == provider.data_source - assert "What did you expect?" == record.exception - - # But the coverage provider did run, and the timestamp is now set to - # a recent value. - value = Timestamp.value( - db.session, - provider.service_name, - service_type=Timestamp.COVERAGE_PROVIDER_TYPE, - collection=None, - ) - assert (utc_now() - value).total_seconds() < 1 - - def test_run_transient_failure(self, db: DatabaseTransactionFixture): - """Verify that TransientFailureCoverageProvider works the - way we'd expect. - """ - - provider = TransientFailureCoverageProvider(db.session) - identifier = db.identifier() - - # We start with no CoverageRecords and no Timestamp. - assert [] == db.session.query(CoverageRecord).all() - assert None == Timestamp.value( - db.session, - provider.service_name, - service_type=Timestamp.COVERAGE_PROVIDER_TYPE, - collection=None, - ) - - now = utc_now() - provider.run() - - # We have a CoverageRecord representing the transient failure. - [failure] = identifier.coverage_records - assert CoverageRecord.TRANSIENT_FAILURE == failure.status - - # The timestamp was set. - timestamp = Timestamp.value( - db.session, - provider.service_name, - service_type=Timestamp.COVERAGE_PROVIDER_TYPE, - collection=None, - ) - assert (timestamp - now).total_seconds() < 1 - - def test_add_coverage_record_for(self, db: DatabaseTransactionFixture): - """Calling CollectionCoverageProvider.add_coverage_record is the same - as calling CoverageRecord.add_for with the relevant - information. - """ - - provider = AlwaysSuccessfulCollectionCoverageProvider(db.default_collection()) - identifier = db.identifier() - record = provider.add_coverage_record_for(identifier) - - # This is the same as calling CoverageRecord.add_for with - # appropriate arguments. - record2, is_new = CoverageRecord.add_for( - identifier, - data_source=provider.data_source, - operation=provider.operation, - collection=provider.collection_or_not, - ) - assert False == is_new - assert record == record2 - - # By default, the CoverageRecord is not associated with any - # particular collection. - assert None == record.collection - - # Setting COVERAGE_COUNTS_FOR_EVERY_COLLECTION to False will - # change that -- a CoverageRecord will only count for the - # collection associated with the CoverageProvider. - provider.COVERAGE_COUNTS_FOR_EVERY_COLLECTION = False - record = provider.add_coverage_record_for(identifier) - assert db.default_collection() == record.collection - - record2, is_new = CoverageRecord.add_for( - identifier, - data_source=provider.data_source, - operation=provider.operation, - collection=provider.collection_or_not, - ) - assert False == is_new - assert record == record2 - - def test_record_failure_as_coverage_record(self): - """TODO: We need test coverage here.""" - - def test_failure(self, db: DatabaseTransactionFixture): - provider = AlwaysSuccessfulCollectionCoverageProvider(db.default_collection()) - identifier = db.identifier() - failure = provider.failure(identifier, error="an error", transient=False) - assert provider.data_source == failure.data_source - assert "an error" == failure.exception - assert False == failure.transient - - # By default, the failure is not associated with any - # particular collection. - assert None == failure.collection - - # Setting COVERAGE_COUNTS_FOR_EVERY_COLLECTION to False - # will change that -- a failure will only count for the - # collection associated with the CoverageProvider. - provider.COVERAGE_COUNTS_FOR_EVERY_COLLECTION = False - failure = provider.failure(identifier, error="an error", transient=False) - assert db.default_collection() == failure.collection - - def test_failure_for_ignored_item(self, db: DatabaseTransactionFixture): - """Test that failure_for_ignored_item creates an appropriate - CoverageFailure. - """ - - identifier = db.identifier() - provider = NeverSuccessfulCoverageProvider(db.session) - result = provider.failure_for_ignored_item(identifier) - assert isinstance(result, CoverageFailure) - assert True == result.transient - assert "Was ignored by CoverageProvider." == result.exception - assert identifier == result.obj - assert provider.data_source == result.data_source - - -class TestCollectionCoverageProvider: - def test_class_variables(self, db: DatabaseTransactionFixture): - """Verify that class variables become appropriate instance - variables. - """ - collection = db.collection(protocol=OPDSAPI) - provider = AlwaysSuccessfulCollectionCoverageProvider(collection) - assert provider.DATA_SOURCE_NAME == provider.data_source.name - - def test_must_have_collection(self): - with pytest.raises(CollectionMissing) as excinfo: - AlwaysSuccessfulCollectionCoverageProvider(None) - assert ( - "AlwaysSuccessfulCollectionCoverageProvider must be instantiated with a Collection." - in str(excinfo.value) - ) - - def test_collection_protocol_must_match_class_protocol( - self, db: DatabaseTransactionFixture - ): - collection = db.collection(protocol=OverdriveAPI) - with pytest.raises(ValueError) as excinfo: - AlwaysSuccessfulCollectionCoverageProvider(collection) - assert ( - "Collection protocol (Overdrive) does not match CoverageProvider protocol (OPDS Import)" - in str(excinfo.value) - ) - - def test_items_that_need_coverage_ignores_collection_when_collection_is_irrelevant( - self, db - ): - # Two providers that do the same work, but one is associated - # with a collection and the other is not. - collection_provider = AlwaysSuccessfulCollectionCoverageProvider( - db.default_collection() - ) - no_collection_provider = AlwaysSuccessfulCoverageProvider(db.session) - - # This distinction is irrelevant because they both consider an - # Identifier covered when it has a CoverageRecord not - # associated with any particular collection. - assert True == collection_provider.COVERAGE_COUNTS_FOR_EVERY_COLLECTION - assert True == no_collection_provider.COVERAGE_COUNTS_FOR_EVERY_COLLECTION - - assert collection_provider.data_source == no_collection_provider.data_source - data_source = collection_provider.data_source - - # Create a license pool belonging to the default collection. - pool = db.licensepool(None, collection=db.default_collection()) - identifier = pool.identifier - - def needs(): - """Returns all items that need coverage from both test - CoverageProviders. - """ - return tuple( - p.items_that_need_coverage().all() - for p in (collection_provider, no_collection_provider) - ) - - # We start out in the state where the identifier appears to need - # coverage from both CoverageProviders. - assert ([identifier], [identifier]) == needs() - - # Add coverage for the default collection, and both - # CoverageProviders still consider the identifier - # uncovered. (This shouldn't happen, but if it does, we don't - # count it.) - db.coverage_record(identifier, data_source, collection=db.default_collection()) - assert ([identifier], [identifier]) == needs() - - # Add coverage not associated with any collection, and both - # CoverageProviders consider it covered. - db.coverage_record(identifier, data_source, collection=None) - assert ([], []) == needs() - - def test_items_that_need_coverage_respects_collection_when_collection_is_relevant( - self, db - ): - # Two providers that do the same work, but are associated - # with different collections. - collection_1_provider = AlwaysSuccessfulCollectionCoverageProvider( - db.default_collection() - ) - collection_2 = db.collection() - collection_2_provider = AlwaysSuccessfulCollectionCoverageProvider(collection_2) - - # And one that does the same work but is not associated with - # any collection. - no_collection_provider = AlwaysSuccessfulCoverageProvider(db.session) - - # The 'collection' distinction is relevant, because these - # CoverageProviders consider an identifier covered only when - # it has a CoverageRecord for _their_ collection. - collection_1_provider.COVERAGE_COUNTS_FOR_EVERY_COLLECTION = False - collection_2_provider.COVERAGE_COUNTS_FOR_EVERY_COLLECTION = False - no_collection_provider.COVERAGE_COUNTS_FOR_EVERY_COLLECTION = False - - assert collection_1_provider.data_source == collection_2_provider.data_source - data_source = collection_1_provider.data_source - - # Create a license pool belonging to the default collection so - # that its Identifier will show up as needing coverage by the - # CoverageProvider that manages that collection. - pool = db.licensepool(None, collection=db.default_collection()) - identifier = pool.identifier - - def needs(): - """Returns all items that need coverage from both test - CoverageProviders. - """ - return tuple( - p.items_that_need_coverage().all() - for p in (collection_1_provider, no_collection_provider) - ) - - # We start out in the state where the identifier needs - # coverage from the CoverageProvider not associated with - # any Collection, and the CoverageProvider associated with - # the Collection where the LicensePool lives. - # - assert ([identifier], [identifier]) == needs() - - # The CoverageProvider associated with a different Collection - # doesn't care about this Identifier, because its Collection - # doesn't include that Identiifer. - assert [] == collection_2_provider.items_that_need_coverage().all() - - # Add coverage for an irrelevant collection, and nothing happens. - db.coverage_record(identifier, data_source, collection=db.collection()) - assert ([identifier], [identifier]) == needs() - - # Add coverage for a relevant collection, and it's treated as - # covered by the provider that uses that collection. - db.coverage_record(identifier, data_source, collection=db.default_collection()) - assert ([], [identifier]) == needs() - - # Add coverage not associated with a collection, and it's - # treated as covered by the provider not associated with - # any collection. - db.coverage_record(identifier, data_source, collection=None) - assert ([], []) == needs() - - def test_replacement_policy(self, db: DatabaseTransactionFixture): - """Unless a different replacement policy is passed in, the - replacement policy is ReplacementPolicy.from_license_source(). - """ - provider = AlwaysSuccessfulCollectionCoverageProvider(db.default_collection()) - assert True == provider.replacement_policy.identifiers - assert True == provider.replacement_policy.formats - - policy = ReplacementPolicy.from_metadata_source() - provider = AlwaysSuccessfulCollectionCoverageProvider( - db.default_collection(), replacement_policy=policy - ) - assert policy == provider.replacement_policy - - def test_all(self, db: DatabaseTransactionFixture): - """Verify that all() gives a sequence of CollectionCoverageProvider - objects, one for each Collection that implements the - appropriate protocol. - """ - opds1 = db.collection(protocol=OPDSAPI) - opds2 = db.collection(protocol=OPDSAPI) - overdrive = db.collection(protocol=OverdriveAPI) - providers = list( - AlwaysSuccessfulCollectionCoverageProvider.all(db.session, batch_size=34) - ) - - # The providers were returned in a random order, but there's one - # for each collection that supports the 'OPDS Import' protocol. - assert 2 == len(providers) - collections = {x.collection for x in providers} - assert {opds1, opds2} == collections - - # The providers are of the appropriate type and the keyword arguments - # passed into all() were propagated to the constructor. - for provider in providers: - assert isinstance(provider, AlwaysSuccessfulCollectionCoverageProvider) - assert 34 == provider.batch_size - - def test_set_circulationdata_errors(self, db: DatabaseTransactionFixture): - """Verify that errors when setting circulation data - are turned into CoverageFailure objects. - """ - provider = AlwaysSuccessfulCollectionCoverageProvider(db.default_collection()) - identifier = db.identifier() - - # No db. - failure = provider._set_circulationdata(identifier, None) - assert "Did not receive circulationdata from input source" == failure.exception - data_source_name = provider.data_source.name - - # No identifier in CirculationData. - empty = CirculationData( - data_source_name=data_source_name, primary_identifier_data=None - ) - failure = provider._set_circulationdata(identifier, empty) - assert ( - "Identifier did not match CirculationData's primary identifier." - == failure.exception - ) - - # Mismatched identifier in CirculationData. - wrong = CirculationData( - data_source_name=data_source_name, - primary_identifier_data=IdentifierData.from_identifier(db.identifier()), - ) - failure = provider._set_circulationdata(identifier, empty) - assert ( - "Identifier did not match CirculationData's primary identifier." - == failure.exception - ) - - # Here, the data is okay, but the ReplacementPolicy is - # going to cause an error the first time we try to use it. - correct = CirculationData( - data_source_name=data_source_name, - primary_identifier_data=IdentifierData.from_identifier(identifier), - ) - provider.replacement_policy = object() - failure = provider._set_circulationdata(identifier, correct) - assert isinstance(failure, CoverageFailure) - - # Verify that the general error handling works whether or not - # the provider is associated with a Collection. - provider.collection_id = None - failure = provider._set_circulationdata(identifier, correct) - assert isinstance(failure, CoverageFailure) - - def test_set_bibliographic_incorporates_replacement_policy( - self, db: DatabaseTransactionFixture - ): - # Make sure that if a ReplacementPolicy is passed in to - # set_bibliographic(), the policy's settings (and those of its - # .presentation_calculation_policy) are respected. - # - # This is tested in this class rather than in - # TestIdentifierCoverageProvider because with a collection in - # place we can test a lot more aspects of the ReplacementPolicy. - - edition, pool = db.edition(with_license_pool=True) - identifier = edition.primary_identifier - - class Tripwire: - # This class sets a variable if one of its properties is - # accessed. - def __init__(self, *args, **kwargs): - self.tripped = False - - def __getattr__(self, name): - self.tripped = True - if name.startswith("equivalent_identifier_"): - # These need to be numbers rather than booleans, - # but the exact number doesn't matter. - return 100 - return True - - presentation_calculation_policy = Tripwire() - replacement_policy = ReplacementPolicy.model_construct( - presentation_calculation_policy=cast( - PresentationCalculationPolicy, presentation_calculation_policy - ), - ) - - provider = AlwaysSuccessfulCollectionCoverageProvider( - db.default_collection(), replacement_policy=replacement_policy - ) - - bibliographic = BibliographicData( - data_source_name=provider.data_source.name, - primary_identifier_data=IdentifierData.from_identifier(identifier), - ) - # We've got a CirculationData object that includes an open-access download. - link = LinkData(rel=Hyperlink.OPEN_ACCESS_DOWNLOAD, href="http://foo.com/") - - # We get an error if the CirculationData's identifier is - # doesn't match what we pass in. - circulationdata = CirculationData( - data_source_name=provider.data_source.name, - primary_identifier_data=IdentifierData.from_identifier(db.identifier()), - links=[link], - ) - failure = provider.set_bibliographic_and_circulation_data( - identifier, bibliographic, circulationdata - ) - assert ( - "Identifier did not match CirculationData's primary identifier." - == failure.exception - ) - - # Otherwise, the data is applied. - circulationdata = CirculationData( - data_source_name=provider.data_source.name, - primary_identifier_data=bibliographic.primary_identifier_data, - links=[link], - ) - - provider.set_bibliographic_and_circulation_data( - identifier, bibliographic, circulationdata - ) - - # Our custom PresentationCalculationPolicy was used when - # determining whether to recalculate the work's - # presentation. We know this because the tripwire was - # triggered. - assert True == presentation_calculation_policy.tripped - - def test_items_that_need_coverage(self, db: DatabaseTransactionFixture): - # Here's an Identifier that was covered on 01/01/2016. - identifier = db.identifier() - cutoff_time = datetime_utc(2016, 1, 1) - provider = AlwaysSuccessfulCoverageProvider(db.session) - record, is_new = CoverageRecord.add_for( - identifier, provider.data_source, timestamp=cutoff_time - ) - - # Since the Identifier was covered, it doesn't show up in - # items_that_need_coverage. - assert [] == provider.items_that_need_coverage().all() - - # If we set the CoverageProvider's cutoff_time to the time of - # coverage, the Identifier is still treated as covered. - provider = AlwaysSuccessfulCoverageProvider(db.session, cutoff_time=cutoff_time) - assert [] == provider.items_that_need_coverage().all() - - # But if we set the cutoff time to immediately after the time - # the Identifier was covered... - one_second_after = cutoff_time + datetime.timedelta(seconds=1) - provider = AlwaysSuccessfulCoverageProvider( - db.session, cutoff_time=one_second_after - ) - - # The identifier is treated as lacking coverage. - assert [identifier] == provider.items_that_need_coverage().all() - - def test_work(self, db: DatabaseTransactionFixture): - """Verify that a CollectionCoverageProvider can create a Work.""" - # Here's an Gutenberg ID. - identifier = db.identifier(identifier_type=Identifier.GUTENBERG_ID) - - # Here's a CollectionCoverageProvider that is associated - # with an OPDS import-style Collection. - provider = AlwaysSuccessfulCollectionCoverageProvider(db.default_collection()) - - # This CoverageProvider cannot create a Work for the given - # Identifier, because that would require creating a - # LicensePool, and work() won't create a LicensePool if one - # doesn't already exist. - result = provider.work(identifier) - assert isinstance(result, CoverageFailure) - assert "Cannot locate LicensePool" == result.exception - - # The CoverageProvider _can_ automatically create a - # LicensePool, but since there is no Edition associated with - # the Identifier, a Work still can't be created. - pool = provider.license_pool(identifier) - result = provider.work(identifier) - assert isinstance(result, CoverageFailure) - assert "Work could not be calculated" == result.exception - - # So let's use the CoverageProvider to create an Edition - # with minimal bibliographic information. - edition = provider.edition(identifier) - edition.title = "A title" - - # Now we can create a Work. - work = provider.work(identifier) - assert isinstance(work, Work) - assert "A title" == work.title - - # If necessary, we can tell work() to use a specific - # LicensePool when calculating the Work. This is an extreme - # example in which the LicensePool to use has a different - # Identifier (identifier2) than the Identifier we're - # processing (identifier1). - # - # In a real case, this would be used by a CoverageProvider - # that just had to create a LicensePool using an - # INTERNAL_PROCESSING DataSource rather than the DataSource - # associated with the CoverageProvider. - db.session.delete(pool) - identifier2 = db.identifier() - identifier.licensed_through = [] - collection2 = db.collection() - edition2 = db.edition( - identifier_type=identifier2.type, identifier_id=identifier2.identifier - ) - pool2 = db.licensepool(edition=edition2, collection=collection2) - work2 = provider.work(identifier, pool2) - assert work2 != work - assert [pool2] == work2.license_pools - - # Once an identifier has a work associated with it, - # that's always the one that's used, and the value of license_pool - # is ignored. - work3 = provider.work(identifier2, object()) - assert work2 == work3 - - # Any keyword arguments passed into work() are propagated to - # calculate_work(). This lets use (e.g.) create a Work even - # when there is no title. - edition, pool = db.edition(with_license_pool=True) - edition.title = None - work = provider.work(pool.identifier, pool, even_if_no_title=True) - assert isinstance(work, Work) - assert None == work.title - - # If a work exists but is not presentation-ready, - # CollectionCoverageProvider.work() will call calculate_work() - # in an attempt to fix it. - edition.title = "Finally a title" - work2 = provider.work(pool.identifier, pool) - assert work2 == work - assert "Finally a title" == work.title - assert True == work.presentation_ready - - # Once the work is presentation_ready, calling - # CollectionCoverageProvider.work() will no longer call - # calculate_work() -- it will just return the work. - def explode(): - raise Exception("don't call me!") - - pool.calculate_work = explode - work2 = provider.work(pool.identifier, pool) - assert work2 == work - - def test_set_bibliographic_and_circulation_data( - self, - bibliographic_data, - circulation_data, - db, - ): - """Verify that a CollectionCoverageProvider can set both - bibliographic data (on an Edition) and circulation data (on a LicensePool). - """ - # Here's an Overdrive Identifier to work with. - identifier = db.identifier( - identifier_type=Identifier.OVERDRIVE_ID, - foreign_id=bibliographic_data.primary_identifier_data.identifier, - ) - - # Here's a CollectionCoverageProvider that is associated with - # an Overdrive-type Collection. (We have to subclass and talk - # about Overdrive because BIBLIOGRAPHIC_DATA and - # CIRCULATION_DATA are data for an Overdrive book.) - class OverdriveProvider(AlwaysSuccessfulCollectionCoverageProvider): - DATA_SOURCE_NAME = DataSource.OVERDRIVE - PROTOCOL = OverdriveAPI.label() - IDENTIFIER_TYPES = Identifier.OVERDRIVE_ID - - collection = db.collection(protocol=OverdriveAPI) - provider = OverdriveProvider(collection) - - # We get a CoverageFailure if we don't pass in any data at all. - result = provider.set_bibliographic_and_circulation_data(identifier, None, None) - assert isinstance(result, CoverageFailure) - assert ( - "Received neither bibliographic data nor circulation data from input source" - == result.exception - ) - - # We get a CoverageFailure if no work can be created. In this - # case, that happens because the bibliographic data doesn't provide a - # title. - old_title = bibliographic_data.title - bibliographic_data.title = None - result = provider.set_bibliographic_and_circulation_data( - identifier, bibliographic_data, circulation_data - ) - assert isinstance(result, CoverageFailure) - assert "Work could not be calculated" == result.exception - - # Restore the title and try again. This time it will work. - bibliographic_data.title = old_title - result = provider.set_bibliographic_and_circulation_data( - identifier, bibliographic_data, circulation_data - ) - assert result == identifier - - # An Edition was created to hold the bibliographic data, a LicensePool - # was created to hold the circulation data, and a Work - # was created to bind everything together. - [edition] = identifier.primarily_identifies - assert "A Girl Named Disaster" == edition.title - [pool] = identifier.licensed_through - work = identifier.work - assert work == pool.work - - # CoverageProviders that offer bibliographic information - # typically don't have circulation information in the sense of - # 'how many copies are in this Collection?', but sometimes - # they do have circulation information in the sense of 'what - # formats are available?' - [lpdm] = pool.delivery_mechanisms - mechanism = lpdm.delivery_mechanism - assert "application/epub+zip (DRM-free)" == mechanism.name - - # If there's an exception setting the bibliographic data, a - # CoverageFailure results. This call raises a ValueError - # because the identifier we're trying to cover doesn't match - # the identifier found in the BibliographicData object. - old_identifier = bibliographic_data.primary_identifier_data - bibliographic_data.primary_identifier_data = IdentifierData( - type=Identifier.OVERDRIVE_ID, identifier="abcd" - ) - result = provider.set_bibliographic_and_circulation_data( - identifier, bibliographic_data, circulation_data - ) - assert isinstance(result, CoverageFailure) - assert "ValueError" in result.exception - bibliographic_data.primary_identifier_data = old_identifier - - def test_autocreate_licensepool(self, db: DatabaseTransactionFixture): - """A CollectionCoverageProvider can locate (or, if necessary, create) - a LicensePool for an identifier. - """ - identifier = db.identifier() - assert [] == identifier.licensed_through - provider = AlwaysSuccessfulCollectionCoverageProvider(db.default_collection()) - pool = provider.license_pool(identifier) - assert [pool] == identifier.licensed_through - assert pool.data_source == provider.data_source - assert pool.identifier == identifier - assert pool.collection == provider.collection - - # Calling license_pool again finds the same LicensePool - # as before. - pool2 = provider.license_pool(identifier) - assert pool == pool2 - - # It's possible for a CollectionCoverageProvider to create a - # LicensePool for a different DataSource than the one - # associated with the Collection. Only the metadata wrangler - # needs to do this -- it's so a CoverageProvider for a - # third-party DataSource can create an 'Internal Processing' - # LicensePool when some other part of the metadata wrangler - # failed to do this earlier. - - # If a working pool already exists, it's returned and no new - # pool is created. - same_pool = provider.license_pool(identifier, DataSource.INTERNAL_PROCESSING) - assert same_pool == pool2 - assert provider.data_source == same_pool.data_source - - # A new pool is only created if no working pool can be found. - identifier2 = db.identifier() - new_pool = provider.license_pool(identifier2, DataSource.INTERNAL_PROCESSING) - assert new_pool.data_source.name == DataSource.INTERNAL_PROCESSING - assert new_pool.identifier == identifier2 - assert new_pool.collection == provider.collection - - def test_set_presentation_ready(self, db: DatabaseTransactionFixture): - """Test that a CollectionCoverageProvider can set a Work - as presentation-ready. - """ - identifier = db.identifier() - provider = AlwaysSuccessfulCollectionCoverageProvider(db.default_collection()) - - # If there is no LicensePool for the Identifier, - # set_presentation_ready will not try to create one, - # and so no Work will be created. - result = provider.set_presentation_ready(identifier) - assert isinstance(result, CoverageFailure) - assert "Cannot locate LicensePool" == result.exception - - # Once a LicensePool and a suitable Edition exist, - # set_presentation_ready will create a Work for the item and - # mark it presentation ready. - pool = provider.license_pool(identifier) - edition = provider.edition(identifier) - edition.title = "A title" - result = provider.set_presentation_ready(identifier) - assert result == identifier - assert True == pool.work.presentation_ready - - -class BibliographicCoverageProviderFixture: - transaction: DatabaseTransactionFixture - work: Work - pool: LicensePool - identifier: Identifier - - @classmethod - def create( - cls, transaction: DatabaseTransactionFixture - ) -> "BibliographicCoverageProviderFixture": - data = BibliographicCoverageProviderFixture() - data.transaction = transaction - data.work = transaction.work( - with_license_pool=True, with_open_access_download=True - ) - data.work.presentation_ready = False - [data.pool] = data.work.license_pools - data.identifier = data.pool.identifier - return data - - -@pytest.fixture -def bibliographic_coverage_provider_fixture( - db, -) -> BibliographicCoverageProviderFixture: - return BibliographicCoverageProviderFixture.create(db) - - -class TestBibliographicCoverageProvider: - """Test the features specific to BibliographicCoverageProvider.""" - - def test_work_set_presentation_ready_on_success( - self, - bibliographic_coverage_provider_fixture: BibliographicCoverageProviderFixture, - ): - data = bibliographic_coverage_provider_fixture - - # When a Work is successfully run through a - # BibliographicCoverageProvider, it's set as presentation-ready. - provider = AlwaysSuccessfulBibliographicCoverageProvider(data.pool.collection) - [result] = provider.process_batch([data.identifier]) - assert result == data.identifier - assert True == data.work.presentation_ready - - # ensure_coverage does the same thing. - data.work.presentation_ready = False - result = provider.ensure_coverage(data.identifier) - assert isinstance(result, CoverageRecord) - assert result.identifier == data.identifier - assert True == data.work.presentation_ready - - def test_failure_does_not_set_work_presentation_ready( - self, - bibliographic_coverage_provider_fixture: BibliographicCoverageProviderFixture, - ): - """A Work is not set as presentation-ready except on success.""" - data = bibliographic_coverage_provider_fixture - - provider = NeverSuccessfulBibliographicCoverageProvider(data.pool.collection) - result = provider.ensure_coverage(data.identifier) - assert CoverageRecord.TRANSIENT_FAILURE == result.status - assert False == data.work.presentation_ready - - -class MockWork: - """A Work-like object that keeps track of the policy that was used - to recalculate its presentation. - """ - - def calculate_presentation(self, policy): - self.calculate_presentation_called_with = policy diff --git a/tests/manager/data_layer/test_bibliographic.py b/tests/manager/data_layer/test_bibliographic.py index 8d22db9a18..098ce8d728 100644 --- a/tests/manager/data_layer/test_bibliographic.py +++ b/tests/manager/data_layer/test_bibliographic.py @@ -6,7 +6,6 @@ import pytest from freezegun import freeze_time -from sqlalchemy import select from palace.util.datetime_helpers import utc_now from palace.util.exceptions import PalaceValueError @@ -25,7 +24,6 @@ from palace.manager.data_layer.subject import SubjectData from palace.manager.sqlalchemy.model.classification import Subject from palace.manager.sqlalchemy.model.contributor import Contributor -from palace.manager.sqlalchemy.model.coverage import CoverageRecord from palace.manager.sqlalchemy.model.datasource import DataSource from palace.manager.sqlalchemy.model.edition import Edition from palace.manager.sqlalchemy.model.identifier import Identifier @@ -714,58 +712,6 @@ def test_apply_no_value(self, db: DatabaseTransactionFixture): assert edition_new.published == edition_old.published assert edition_new.issued == edition_old.issued - def test_apply_creates_coverage_records(self, db: DatabaseTransactionFixture): - edition, pool = db.edition(with_license_pool=True) - - bibliographic = BibliographicData( - data_source_name=DataSource.OVERDRIVE, title=db.fresh_str() - ) - - edition, changed = bibliographic.apply(db.session, edition, pool.collection) - - # One success was recorded. - records = ( - db.session.query(CoverageRecord) - .filter(CoverageRecord.identifier_id == edition.primary_identifier.id) - .filter(CoverageRecord.operation == None) - ) - assert 1 == records.count() - assert CoverageRecord.SUCCESS == records.all()[0].status - - # Apply BibliographicData from a different source. - bibliographic = BibliographicData( - data_source_name=DataSource.GUTENBERG, title=db.fresh_str() - ) - - edition, changed = bibliographic.apply(db.session, edition, pool.collection) - - # Another success record was created. - records = ( - db.session.query(CoverageRecord) - .filter(CoverageRecord.identifier_id == edition.primary_identifier.id) - .filter(CoverageRecord.operation == None) - ) - assert 2 == records.count() - for record in records.all(): - assert CoverageRecord.SUCCESS == record.status - - def test_apply_does_not_create_coverage_records( - self, db: DatabaseTransactionFixture - ): - edition, pool = db.edition(with_license_pool=True) - - bibliographic = BibliographicData( - data_source_name=DataSource.OVERDRIVE, title=db.fresh_str() - ) - - bibliographic.apply( - db.session, edition, pool.collection, create_coverage_record=False - ) - - # No coverage records were created. - records = db.session.scalars(select(CoverageRecord)).all() - assert len(records) == 0 - def test_apply_no_changes_needed(self, db: DatabaseTransactionFixture): edition, pool = db.edition(with_license_pool=True) edition.title = "Old title" diff --git a/tests/manager/integration/license/overdrive/test_api.py b/tests/manager/integration/license/overdrive/test_api.py index 12377cc914..5510cdccad 100644 --- a/tests/manager/integration/license/overdrive/test_api.py +++ b/tests/manager/integration/license/overdrive/test_api.py @@ -1736,10 +1736,10 @@ def test_update_licensepool_provides_bibliographic_coverage( http.queue_response(200, content=bibliographic) # Now we're ready. When we call update_licensepool, the - # OverdriveAPI will retrieve the availability information, - # then the bibliographic information. It will then trigger the - # OverdriveBibliographicCoverageProvider, which will - # create an Edition and a presentation-ready Work. + # OverdriveAPI will retrieve the availability information, then + # the bibliographic information. It will then apply that + # bibliographic data to create an Edition and a + # presentation-ready Work. pool, was_new, changed = overdrive_api_fixture.api.update_licensepool( identifier.identifier ) @@ -1754,15 +1754,6 @@ def test_update_licensepool_provides_bibliographic_coverage( "http://images.contentreserve.com/" ) - # The book has been run through the bibliographic coverage - # provider. - coverage = [ - x - for x in identifier.coverage_records - if x.operation is None and x.data_source.name == DataSource.OVERDRIVE - ] - assert len(coverage) == 1 - # Call update_licensepool on an identifier that is missing a work and make # sure that it provides bibliographic coverage in that case. db.session.delete(pool.work) diff --git a/tests/manager/integration/license/overdrive/test_coverage.py b/tests/manager/integration/license/overdrive/test_coverage.py deleted file mode 100644 index cf46e2b031..0000000000 --- a/tests/manager/integration/license/overdrive/test_coverage.py +++ /dev/null @@ -1,150 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - -import pytest - -from palace.manager.core.coverage import CoverageFailure -from palace.manager.integration.license.overdrive.coverage import ( - OverdriveBibliographicCoverageProvider, -) -from palace.manager.scripts.coverage_provider import RunCollectionCoverageProviderScript -from palace.manager.sqlalchemy.model.identifier import Identifier -from palace.manager.sqlalchemy.model.licensing import DeliveryMechanism -from palace.manager.sqlalchemy.model.resource import Representation -from tests.fixtures.database import DatabaseTransactionFixture -from tests.fixtures.overdrive import OverdriveAPIFixture -from tests.mocks.overdrive import MockOverdriveAPI - - -@dataclass -class OverdriveBibliographicCoverageProviderFixture: - overdrive: OverdriveAPIFixture - provider: OverdriveBibliographicCoverageProvider - api: MockOverdriveAPI - - -@pytest.fixture -def overdrive_biblio_provider_fixture( - db: DatabaseTransactionFixture, - overdrive_api_fixture: OverdriveAPIFixture, -) -> OverdriveBibliographicCoverageProviderFixture: - overdrive = overdrive_api_fixture - api = overdrive_api_fixture.api - provider = OverdriveBibliographicCoverageProvider( - overdrive_api_fixture.collection, api=api - ) - return OverdriveBibliographicCoverageProviderFixture( - overdrive, - provider, - api, - ) - - -class TestOverdriveBibliographicCoverageProvider: - """Test the code that looks up bibliographic information from Overdrive.""" - - def test_script_instantiation( - self, - overdrive_biblio_provider_fixture: OverdriveBibliographicCoverageProviderFixture, - ): - """Test that RunCoverageProviderScript can instantiate - the coverage provider. - """ - - fixture = overdrive_biblio_provider_fixture - db = fixture.overdrive.db - - script = RunCollectionCoverageProviderScript( - OverdriveBibliographicCoverageProvider, - db.session, - api=fixture.api, - ) - [provider] = script.providers - assert isinstance(provider, OverdriveBibliographicCoverageProvider) - assert provider.api is fixture.api - assert fixture.overdrive.collection == provider.collection - - def test_invalid_or_unrecognized_guid( - self, - overdrive_biblio_provider_fixture: OverdriveBibliographicCoverageProviderFixture, - db: DatabaseTransactionFixture, - ): - """A bad or malformed GUID can't get coverage.""" - fixture = overdrive_biblio_provider_fixture - http = fixture.overdrive.mock_http - - identifier = db.identifier() - identifier.identifier = "bad guid" - - error = '{"errorCode": "InvalidGuid", "message": "An invalid guid was given.", "token": "7aebce0e-2e88-41b3-b6d3-82bf15f8e1a2"}' - http.queue_response(200, content=error) - - failure = fixture.provider.process_item(identifier) - assert isinstance(failure, CoverageFailure) - assert failure.transient is False - assert failure.exception == "Invalid Overdrive ID: bad guid" - - # This is for when the GUID is well-formed but doesn't - # correspond to any real Overdrive book. - error = '{"errorCode": "NotFound", "message": "Not found in Overdrive collection.", "token": "7aebce0e-2e88-41b3-b6d3-82bf15f8e1a2"}' - http.queue_response(200, content=error) - - failure = fixture.provider.process_item(identifier) - assert isinstance(failure, CoverageFailure) - assert failure.transient is False - assert failure.exception == "ID not recognized by Overdrive: bad guid" - - def test_process_item_creates_presentation_ready_work( - self, - overdrive_biblio_provider_fixture: OverdriveBibliographicCoverageProviderFixture, - db: DatabaseTransactionFixture, - ): - """Test the normal workflow where we ask Overdrive for data, - Overdrive provides it, and we create a presentation-ready work. - """ - fixture = overdrive_biblio_provider_fixture - http = fixture.overdrive.mock_http - - # Here's the book mentioned in overdrive_metadata.json. - identifier = db.identifier(identifier_type=Identifier.OVERDRIVE_ID) - identifier.identifier = "3896665d-9d81-4cac-bd43-ffc5066de1f5" - - # This book has no LicensePool. - assert identifier.licensed_through == [] - - # Run it through the OverdriveBibliographicCoverageProvider - raw, info = fixture.overdrive.sample_json("overdrive_metadata.json") - http.queue_response(200, content=raw) - - [result] = fixture.provider.process_batch([identifier]) - - assert result == identifier - - # A LicensePool was created, not because we know anything - # about how we've licensed this book, but to have a place to - # store the information about what formats the book is - # available in. - [pool] = identifier.licensed_through - assert pool.licenses_owned == 0 - assert { - ( - x.delivery_mechanism.content_type, - x.delivery_mechanism.drm_scheme, - x.available, - ) - for x in pool.delivery_mechanisms - } == { - (Representation.EPUB_MEDIA_TYPE, DeliveryMechanism.NO_DRM, False), - (Representation.PDF_MEDIA_TYPE, DeliveryMechanism.NO_DRM, False), - (Representation.EPUB_MEDIA_TYPE, DeliveryMechanism.ADOBE_DRM, True), - ( - DeliveryMechanism.STREAMING_TEXT_CONTENT_TYPE, - DeliveryMechanism.STREAMING_DRM, - True, - ), - } - - # A Work was created and made presentation ready. - assert pool.work.title == "Agile Documentation" - assert pool.work.presentation_ready is True diff --git a/tests/manager/integration/license/test_bibliotheca.py b/tests/manager/integration/license/test_bibliotheca.py index ea4075fb0b..683f24af26 100644 --- a/tests/manager/integration/license/test_bibliotheca.py +++ b/tests/manager/integration/license/test_bibliotheca.py @@ -3,7 +3,7 @@ import json import random from datetime import date, timedelta -from typing import TYPE_CHECKING, cast +from typing import TYPE_CHECKING from unittest.mock import create_autospec, patch import pytest @@ -32,7 +32,6 @@ from palace.manager.celery.tasks import apply from palace.manager.integration.license.bibliotheca import ( BibliothecaAPI, - BibliothecaBibliographicCoverageProvider, BibliothecaParser, CheckoutResponseParser, ErrorParser, @@ -40,7 +39,6 @@ ItemListParser, PatronCirculationParser, ) -from palace.manager.scripts.coverage_provider import RunCollectionCoverageProviderScript from palace.manager.sqlalchemy.model.circulationevent import CirculationEvent from palace.manager.sqlalchemy.model.classification import Subject from palace.manager.sqlalchemy.model.contributor import Contributor @@ -1199,62 +1197,6 @@ def test_circulation_data_status_exhausted( assert circulation.licenses_owned == 0 assert circulation.status == LicensePoolStatus.EXHAUSTED - -class TestBibliographicCoverageProvider(TestBibliothecaAPI): - """Test the code that looks up bibliographic information from Bibliotheca.""" - - def test_script_instantiation(self, bibliotheca_fixture: BibliothecaAPITestFixture): - """Test that RunCollectionCoverageProviderScript can instantiate - this coverage provider. - """ - script = RunCollectionCoverageProviderScript( - BibliothecaBibliographicCoverageProvider, - bibliotheca_fixture.db.session, - api_class=MockBibliothecaAPI, - ) - [provider] = script.providers - assert isinstance(provider, BibliothecaBibliographicCoverageProvider) - assert isinstance(provider.api, MockBibliothecaAPI) - - def test_process_item_creates_presentation_ready_work( - self, bibliotheca_fixture: BibliothecaAPITestFixture - ): - db = bibliotheca_fixture.db - # Test the normal workflow where we ask Bibliotheca for data, - # Bibliotheca provides it, and we create a presentation-ready work. - identifier = db.identifier(identifier_type=Identifier.BIBLIOTHECA_ID) - identifier.identifier = "ddf4gr9" - - # This book has no LicensePools. - assert [] == identifier.licensed_through - - # Run it through the BibliothecaBibliographicCoverageProvider - provider = BibliothecaBibliographicCoverageProvider( - bibliotheca_fixture.collection, api_class=MockBibliothecaAPI - ) - api = cast(MockBibliothecaAPI, provider.api) - data = bibliotheca_fixture.files.sample_data("item_metadata_single.xml") - - # We can't use bibliotheca_fixture.api because that's not the same object - # as the one created by the coverage provider. - api.queue_response(200, content=data) - [result] = provider.process_batch([identifier]) - assert identifier == result - # A LicensePool was created and populated with format and availability - # information. - [pool] = identifier.licensed_through - assert 1 == pool.licenses_owned - assert 1 == pool.licenses_available - [lpdm] = pool.delivery_mechanisms - assert ( - "application/epub+zip (application/vnd.adobe.adept+xml)" - == lpdm.delivery_mechanism.name - ) - - # A Work was created and made presentation ready. - assert "The Incense Game" == pool.work.title - assert True == pool.work.presentation_ready - def test_internal_formats(self): m = ItemListParser.internal_formats diff --git a/tests/manager/integration/license/test_bibliotheca_circulation_updater.py b/tests/manager/integration/license/test_bibliotheca_circulation_updater.py index 4bcbd9357c..694a4bbaa7 100644 --- a/tests/manager/integration/license/test_bibliotheca_circulation_updater.py +++ b/tests/manager/integration/license/test_bibliotheca_circulation_updater.py @@ -351,7 +351,6 @@ def test_changed_title_applied_synchronously( # The change was applied synchronously, not dispatched to a worker. mock_bib.apply.assert_called_once() - assert mock_bib.apply.call_args.kwargs["create_coverage_record"] is False mock_apply.delay.assert_not_called() def test_unchanged_title_is_not_applied( diff --git a/tests/manager/scripts/test_coverage_provider.py b/tests/manager/scripts/test_coverage_provider.py deleted file mode 100644 index a27c299628..0000000000 --- a/tests/manager/scripts/test_coverage_provider.py +++ /dev/null @@ -1,27 +0,0 @@ -from __future__ import annotations - -from palace.util.datetime_helpers import datetime_utc - -from palace.manager.scripts.coverage_provider import ( - RunCoverageProviderScript, -) -from tests.fixtures.database import DatabaseTransactionFixture -from tests.mocks.stdin import MockStdin - - -class TestRunCoverageProviderScript: - def test_parse_command_line(self, db: DatabaseTransactionFixture): - identifier = db.identifier() - cmd_args = [ - "--cutoff-time", - "2016-05-01", - "--identifier-type", - identifier.type, - identifier.identifier, - ] - parsed = RunCoverageProviderScript.parse_command_line( - db.session, cmd_args, MockStdin() - ) - assert datetime_utc(2016, 5, 1) == parsed.cutoff_time - assert [identifier] == parsed.identifiers - assert identifier.type == parsed.identifier_type diff --git a/tests/manager/scripts/test_informational.py b/tests/manager/scripts/test_informational.py index 2343af1406..f6cb6870ee 100644 --- a/tests/manager/scripts/test_informational.py +++ b/tests/manager/scripts/test_informational.py @@ -16,8 +16,6 @@ WhereAreMyBooksScript, ) from palace.manager.search.external_search import ExternalSearchIndex -from palace.manager.sqlalchemy.model.coverage import CoverageRecord -from palace.manager.sqlalchemy.model.datasource import DataSource from palace.manager.sqlalchemy.model.licensing import LicensePoolStatus from tests.fixtures.database import DatabaseTransactionFixture from tests.fixtures.search import EndToEndSearchFixture @@ -495,8 +493,6 @@ def test_explain(self, db: DatabaseTransactionFixture): [pool] = work.license_pools edition = work.presentation_edition identifier = pool.identifier - source = DataSource.lookup(db.session, DataSource.OCLC_LINKED_DATA) - CoverageRecord.add_for(identifier, source, "an operation") input = StringIO() io_output = StringIO() args = ["--identifier-type", "Database ID", str(identifier.id)] @@ -513,10 +509,6 @@ def test_explain(self, db: DatabaseTransactionFixture): for contributor in edition.contributors: assert contributor.sort_name in output - # CoverageRecords associated with the primary identifier were - # printed out. - assert "OCLC Linked Data | an operation | success" in output - # There is an active LicensePool that is fulfillable and has # copies owned. assert "%s owned" % pool.licenses_owned in output diff --git a/tests/manager/sqlalchemy/model/test_collection.py b/tests/manager/sqlalchemy/model/test_collection.py index 03ac15b962..538338c5c7 100644 --- a/tests/manager/sqlalchemy/model/test_collection.py +++ b/tests/manager/sqlalchemy/model/test_collection.py @@ -22,7 +22,6 @@ ) from palace.manager.sqlalchemy.model.circulationevent import CirculationEvent from palace.manager.sqlalchemy.model.collection import Collection -from palace.manager.sqlalchemy.model.coverage import CoverageRecord from palace.manager.sqlalchemy.model.customlist import CustomList from palace.manager.sqlalchemy.model.datasource import DataSource from palace.manager.sqlalchemy.model.edition import Edition @@ -656,16 +655,6 @@ def test_delete( pool2 = db.licensepool(None, collection=collection2) work2.license_pools.append(pool2) - record, _ = CoverageRecord.add_for( - work.presentation_edition, collection.data_source, collection=collection - ) - assert ( - CoverageRecord.lookup( - work.presentation_edition, collection.data_source, collection=collection - ) - != None - ) - # If we're meant to test an inactive collection, make it inactive. if is_inactive: db.make_collection_inactive(collection) @@ -695,9 +684,6 @@ def test_delete( # The default library now has no collections. assert [] == library.associated_collections - # The collection based coverage record got deleted - assert db.session.get(CoverageRecord, record.id) == None - # The deletion of the Collection's sole LicensePool has # cascaded to Loan, Hold, License, and # CirculationEvent. diff --git a/tests/manager/sqlalchemy/model/test_coverage.py b/tests/manager/sqlalchemy/model/test_coverage.py index af15c463fc..ec4911fdc4 100644 --- a/tests/manager/sqlalchemy/model/test_coverage.py +++ b/tests/manager/sqlalchemy/model/test_coverage.py @@ -1,20 +1,14 @@ import datetime from unittest.mock import MagicMock -import pytest from freezegun import freeze_time from palace.util.datetime_helpers import datetime_utc, utc_now from palace.manager.core.monitor import TimestampData from palace.manager.sqlalchemy.model.coverage import ( - BaseCoverageRecord, - CoverageRecord, Timestamp, ) -from palace.manager.sqlalchemy.model.datasource import DataSource -from palace.manager.sqlalchemy.model.edition import Edition -from palace.manager.sqlalchemy.model.identifier import Identifier from palace.manager.util.sentinel import SentinelType from tests.fixtures.database import DatabaseTransactionFixture @@ -229,312 +223,3 @@ def test_recording(self) -> None: assert stamp.start == now assert stamp.finish == now + delta assert stamp.exception == "testing" - - -class TestBaseCoverageRecord: - def test_not_covered(self, db: DatabaseTransactionFixture): - source = DataSource.lookup(db.session, DataSource.OCLC) - - # Here are four identifiers with four relationships to a - # certain coverage provider: no coverage at all, successful - # coverage, a transient failure and a permanent failure. - - no_coverage = db.identifier() - - success = db.identifier() - success_record = db.coverage_record(success, source) - success_record.timestamp = utc_now() - datetime.timedelta(seconds=3600) - assert CoverageRecord.SUCCESS == success_record.status - - transient = db.identifier() - transient_record = db.coverage_record( - transient, source, status=CoverageRecord.TRANSIENT_FAILURE - ) - assert CoverageRecord.TRANSIENT_FAILURE == transient_record.status - - persistent = db.identifier() - persistent_record = db.coverage_record( - persistent, source, status=BaseCoverageRecord.PERSISTENT_FAILURE - ) - assert CoverageRecord.PERSISTENT_FAILURE == persistent_record.status - - # Here's a query that finds all four. - qu = db.session.query(Identifier).outerjoin(CoverageRecord) - assert 4 == qu.count() - - def check_not_covered(expect, **kwargs): - missing = CoverageRecord.not_covered(**kwargs) - assert sorted(expect) == sorted(qu.filter(missing).all()) - - # By default, not_covered() only finds the identifier with no - # coverage and the one with a transient failure. - check_not_covered([no_coverage, transient]) - - # If we pass in different values for covered_status, we change what - # counts as 'coverage'. In this case, we allow transient failures - # to count as 'coverage'. - check_not_covered( - [no_coverage], - count_as_covered=[ - CoverageRecord.PERSISTENT_FAILURE, - CoverageRecord.TRANSIENT_FAILURE, - CoverageRecord.SUCCESS, - ], - ) - - # Here, only success counts as 'coverage'. - check_not_covered( - [no_coverage, transient, persistent], - count_as_covered=CoverageRecord.SUCCESS, - ) - - # We can also say that coverage doesn't count if it was achieved before - # a certain time. Here, we'll show that passing in the timestamp - # of the 'success' record means that record still counts as covered. - check_not_covered( - [no_coverage, transient], - count_as_not_covered_if_covered_before=success_record.timestamp, - ) - - # But if we pass in a time one second later, the 'success' - # record no longer counts as covered. - assert isinstance(success_record.timestamp, datetime.datetime) - one_second_after = success_record.timestamp + datetime.timedelta(seconds=1) - check_not_covered( - [success, no_coverage, transient], - count_as_not_covered_if_covered_before=one_second_after, - ) - - -class TestCoverageRecord: - def test_lookup(self, db: DatabaseTransactionFixture): - source = DataSource.lookup(db.session, DataSource.OCLC) - edition = db.edition() - operation = "foo" - collection = db.default_collection() - record = db.coverage_record(edition, source, operation, collection=collection) - - # To find the CoverageRecord, edition, source, operation, - # and collection must all match. - result = CoverageRecord.lookup( - edition, source, operation, collection=collection - ) - assert record == result - - # You can substitute the Edition's primary identifier for the - # Edition iteslf. - lookup = CoverageRecord.lookup( - edition.primary_identifier, - source, - operation, - collection=db.default_collection(), - ) - assert lookup == record - - # Omit the collection, and you find nothing. - result = CoverageRecord.lookup(edition, source, operation) - assert None == result - - # Same for operation. - result = CoverageRecord.lookup(edition, source, collection=collection) - assert None == result - - result = CoverageRecord.lookup( - edition, source, "other operation", collection=collection - ) - assert None == result - - # Same for data source. - other_source = DataSource.lookup(db.session, DataSource.OVERDRIVE) - result = CoverageRecord.lookup( - edition, other_source, operation, collection=collection - ) - assert None == result - - def test_add_for(self, db: DatabaseTransactionFixture): - source = DataSource.lookup(db.session, DataSource.OCLC) - edition = db.edition() - operation = "foo" - record, is_new = CoverageRecord.add_for(edition, source, operation) - assert True == is_new - - # If we call add_for again we get the same record back, but we - # can modify the timestamp. - a_week_ago = utc_now() - datetime.timedelta(days=7) - record2, is_new = CoverageRecord.add_for(edition, source, operation, a_week_ago) - assert record == record2 - assert False == is_new - assert a_week_ago == record2.timestamp - - # If we don't specify an operation we get a totally different - # record. - record3, ignore = CoverageRecord.add_for(edition, source) - assert record3 != record - assert None == record3.operation - seconds = (utc_now() - record3.timestamp).seconds - assert seconds < 10 - - # If we call lookup we get the same record. - record4 = CoverageRecord.lookup(edition.primary_identifier, source) - assert record3 == record4 - - # We can change the status. - record5, is_new = CoverageRecord.add_for( - edition, source, operation, status=CoverageRecord.PERSISTENT_FAILURE - ) - assert record5 == record - assert CoverageRecord.PERSISTENT_FAILURE == record.status - - def test_bulk_add(self, db: DatabaseTransactionFixture): - source = DataSource.lookup(db.session, DataSource.GUTENBERG) - operation = "testing" - - # An untouched identifier. - i1 = db.identifier() - - # An identifier that already has failing coverage. - covered = db.identifier() - existing = db.coverage_record( - covered, - source, - operation=operation, - status=CoverageRecord.TRANSIENT_FAILURE, - exception="Uh oh", - ) - original_timestamp = existing.timestamp - - resulting_records, ignored_identifiers = CoverageRecord.bulk_add( - [i1, covered], source, operation=operation - ) - - # A new coverage record is created for the uncovered identifier. - assert i1.coverage_records == resulting_records - [new_record] = resulting_records - assert source == new_record.data_source - assert operation == new_record.operation - assert CoverageRecord.SUCCESS == new_record.status - assert None == new_record.exception - - # The existing coverage record is untouched. - assert [covered] == ignored_identifiers - assert [existing] == covered.coverage_records - assert CoverageRecord.TRANSIENT_FAILURE == existing.status - assert original_timestamp == existing.timestamp - assert "Uh oh" == existing.exception - - # Newly untouched identifier. - i2 = db.identifier() - - # Force bulk add. - resulting_records, ignored_identifiers = CoverageRecord.bulk_add( - [i2, covered], source, operation=operation, force=True - ) - - # The new identifier has the expected coverage. - [new_record] = i2.coverage_records - assert new_record in resulting_records - - # The existing record has been updated. - assert existing in resulting_records - assert covered not in ignored_identifiers - assert CoverageRecord.SUCCESS == existing.status - assert isinstance(existing.timestamp, datetime.datetime) - assert isinstance(original_timestamp, datetime.datetime) - assert existing.timestamp > original_timestamp - assert None == existing.exception - - # If no records are created or updated, no records are returned. - resulting_records, ignored_identifiers = CoverageRecord.bulk_add( - [i2, covered], source, operation=operation - ) - - assert [] == resulting_records - assert sorted([i2, covered]) == sorted(ignored_identifiers) - - def test_bulk_add_with_collection(self, db: DatabaseTransactionFixture): - source = DataSource.lookup(db.session, DataSource.GUTENBERG) - operation = "testing" - - c1 = db.collection() - c2 = db.collection() - - # An untouched identifier. - i1 = db.identifier() - - # An identifier with coverage for a different collection. - covered = db.identifier() - existing = db.coverage_record( - covered, - source, - operation=operation, - status=CoverageRecord.TRANSIENT_FAILURE, - collection=c1, - exception="Danger, Will Robinson", - ) - original_timestamp = existing.timestamp - - resulting_records, ignored_identifiers = CoverageRecord.bulk_add( - [i1, covered], source, operation=operation, collection=c1, force=True - ) - - assert 2 == len(resulting_records) - assert [] == ignored_identifiers - - # A new record is created for the new identifier. - [new_record] = i1.coverage_records - assert new_record in resulting_records - assert source == new_record.data_source - assert operation == new_record.operation - assert CoverageRecord.SUCCESS == new_record.status - assert c1 == new_record.collection - - # The existing record has been updated. - assert existing in resulting_records - assert CoverageRecord.SUCCESS == existing.status - assert isinstance(existing.timestamp, datetime.datetime) - assert isinstance(original_timestamp, datetime.datetime) - assert existing.timestamp > original_timestamp - assert None == existing.exception - - # Bulk add for a different collection. - resulting_records, ignored_identifiers = CoverageRecord.bulk_add( - [covered], - source, - operation=operation, - collection=c2, - status=CoverageRecord.TRANSIENT_FAILURE, - exception="Oh no", - ) - - # A new record has been added to the identifier. - assert existing not in resulting_records - [new_record] = resulting_records - assert covered == new_record.identifier - assert CoverageRecord.TRANSIENT_FAILURE == new_record.status - assert source == new_record.data_source - assert operation == new_record.operation - assert "Oh no" == new_record.exception - - def test_assert_coverage_operation(self, db: DatabaseTransactionFixture): - """Ensure all the methods that should raise errors, do raise the errors""" - edition: Edition = db.edition() - with pytest.raises(ValueError): - CoverageRecord.add_for( - edition, - edition.data_source, - CoverageRecord.IMPORT_OPERATION, - ) - - with pytest.raises(ValueError): - CoverageRecord.lookup( - edition, - edition.data_source, - CoverageRecord.IMPORT_OPERATION, - ) - - with pytest.raises(ValueError): - CoverageRecord.bulk_add( - [edition.primary_identifier], - edition.data_source, - CoverageRecord.IMPORT_OPERATION, - ) diff --git a/tests/manager/sqlalchemy/model/test_identifier.py b/tests/manager/sqlalchemy/model/test_identifier.py index dceca3850b..fdcf23a01a 100644 --- a/tests/manager/sqlalchemy/model/test_identifier.py +++ b/tests/manager/sqlalchemy/model/test_identifier.py @@ -1,4 +1,3 @@ -import datetime from functools import partial from unittest.mock import MagicMock, PropertyMock, create_autospec, patch @@ -514,131 +513,6 @@ def test_licensed_through_collection(self, db: DatabaseTransactionFixture): assert lp2 == identifier.licensed_through_collection(c2) assert None == identifier.licensed_through_collection(c3) - def test_missing_coverage_from(self, db: DatabaseTransactionFixture): - gutenberg = DataSource.lookup(db.session, DataSource.GUTENBERG) - oclc = DataSource.lookup(db.session, DataSource.OCLC) - web = DataSource.lookup(db.session, DataSource.WEB) - - # Here are two Gutenberg records. - g1, ignore = Edition.for_foreign_id( - db.session, gutenberg, Identifier.GUTENBERG_ID, "1" - ) - - g2, ignore = Edition.for_foreign_id( - db.session, gutenberg, Identifier.GUTENBERG_ID, "2" - ) - - # One of them has coverage from OCLC Classify - c1 = db.coverage_record(g1, oclc) - - # The other has coverage from a specific operation on OCLC Classify - c2 = db.coverage_record(g2, oclc, "some operation") - - # Here's a web record, just sitting there. - w, ignore = Edition.for_foreign_id( - db.session, web, Identifier.URI, "http://www.foo.com/" - ) - - # If we run missing_coverage_from we pick up the Gutenberg - # record with no generic OCLC coverage. It doesn't pick up the - # other Gutenberg record, it doesn't pick up the web record, - # and it doesn't pick up the OCLC coverage for a specific - # operation. - [in_gutenberg_but_not_in_oclc] = Identifier.missing_coverage_from( - db.session, [Identifier.GUTENBERG_ID], oclc - ).all() - - assert g2.primary_identifier == in_gutenberg_but_not_in_oclc - - # If we ask about a specific operation, we get the Gutenberg - # record that has coverage for that operation, but not the one - # that has generic OCLC coverage. - - [has_generic_coverage_only] = Identifier.missing_coverage_from( - db.session, [Identifier.GUTENBERG_ID], oclc, "some operation" - ).all() - assert g1.primary_identifier == has_generic_coverage_only - - # We don't put web sites into OCLC, so this will pick up the - # web record (but not the Gutenberg record). - [in_web_but_not_in_oclc] = Identifier.missing_coverage_from( - db.session, [Identifier.URI], oclc - ).all() - assert w.primary_identifier == in_web_but_not_in_oclc - - # We don't use the web as a source of coverage, so this will - # return both Gutenberg records (but not the web record). - assert [g1.primary_identifier.id, g2.primary_identifier.id] == sorted( - x.id - for x in Identifier.missing_coverage_from( - db.session, [Identifier.GUTENBERG_ID], web - ) - ) - - def test_missing_coverage_from_with_collection( - self, db: DatabaseTransactionFixture - ): - gutenberg = DataSource.lookup(db.session, DataSource.GUTENBERG) - identifier = db.identifier() - collection1 = db.default_collection() - collection2 = db.collection() - db.coverage_record(identifier, gutenberg, collection=collection1) - - # The Identifier has coverage in collection 1. - assert ( - [] - == Identifier.missing_coverage_from( - db.session, [identifier.type], gutenberg, collection=collection1 - ).all() - ) - - # It is missing coverage in collection 2. - assert [identifier] == Identifier.missing_coverage_from( - db.session, [identifier.type], gutenberg, collection=collection2 - ).all() - - # If no collection is specified, we look for a CoverageRecord - # that also has no collection specified, and the Identifier is - # not treated as covered. - assert [identifier] == Identifier.missing_coverage_from( - db.session, [identifier.type], gutenberg - ).all() - - def test_missing_coverage_from_with_cutoff_date( - self, db: DatabaseTransactionFixture - ): - gutenberg_ds = DataSource.lookup(db.session, DataSource.GUTENBERG) - - # Here's an Edition with a coverage record from OCLC classify. - gutenberg, ignore = Edition.for_foreign_id( - db.session, gutenberg_ds, Identifier.GUTENBERG_ID, "1" - ) - identifier = gutenberg.primary_identifier - oclc = DataSource.lookup(db.session, DataSource.OCLC) - coverage = db.coverage_record(gutenberg, oclc) - - # The CoverageRecord knows when the coverage was provided. - timestamp = coverage.timestamp - assert isinstance(timestamp, datetime.datetime) - - # If we ask for Identifiers that are missing coverage records - # as of that time, we see nothing. - assert ( - [] - == Identifier.missing_coverage_from( - db.session, [identifier.type], oclc, count_as_missing_before=timestamp - ).all() - ) - - # But if we give a time one second later, the Identifier is - # missing coverage. - assert [identifier] == Identifier.missing_coverage_from( - db.session, - [identifier.type], - oclc, - count_as_missing_before=timestamp + datetime.timedelta(seconds=1), - ).all() - @pytest.mark.parametrize( "identifier_type,identifier,title", [ diff --git a/tests/migration/test_20260624_5ca948100902_drop_coverage_tables.py b/tests/migration/test_20260624_5ca948100902_drop_coverage_tables.py new file mode 100644 index 0000000000..55d588aecb --- /dev/null +++ b/tests/migration/test_20260624_5ca948100902_drop_coverage_tables.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +from pytest_alembic import MigrationContext +from sqlalchemy import inspect +from sqlalchemy.engine import Engine + +REVISION = "5ca948100902" +DOWN_REVISION = "a6c85605404c" + + +def test_drop_coverage_tables( + alembic_runner: MigrationContext, + alembic_engine: Engine, +) -> None: + """The migration drops coveragerecords and equivalentscoveragerecords. + + Stepping the migration down recreates both tables (and the shared + ``coverage_status`` enum); stepping it back up drops them again while + leaving the unrelated ``timestamps`` table in place. + """ + alembic_runner.migrate_down_to(REVISION) + # Step down once more so the tables exist again. + alembic_runner.migrate_down_one() + assert alembic_runner.current == DOWN_REVISION + + tables = set(inspect(alembic_engine).get_table_names()) + assert "coveragerecords" in tables + assert "equivalentscoveragerecords" in tables + + # Apply the drop. + alembic_runner.migrate_up_one() + assert alembic_runner.current == REVISION + + tables = set(inspect(alembic_engine).get_table_names()) + assert "coveragerecords" not in tables + assert "equivalentscoveragerecords" not in tables + # The unrelated timestamps table is left in place. + assert "timestamps" in tables diff --git a/tests/mocks/mock.py b/tests/mocks/mock.py index 7ee5f8eba0..a89e7f8b1d 100644 --- a/tests/mocks/mock.py +++ b/tests/mocks/mock.py @@ -4,13 +4,6 @@ from requests import Request, Response -from palace.manager.core.coverage import ( - BibliographicCoverageProvider, - CollectionCoverageProvider, - IdentifierCoverageProvider, -) -from palace.manager.integration.license.opds.opds1.api import OPDSAPI -from palace.manager.sqlalchemy.model.datasource import DataSource from palace.manager.sqlalchemy.model.resource import HttpResponseTuple @@ -71,118 +64,6 @@ def __getattr__(self, item): return self.__getitem__(item) -class MockCoverageProvider: - """Mixin class for mock CoverageProviders that defines common constants.""" - - SERVICE_NAME: str | None = "Generic mock CoverageProvider" - - # Whenever a CoverageRecord is created, the data_source of that - # record will be Project Gutenberg. - DATA_SOURCE_NAME = DataSource.GUTENBERG - - # For testing purposes, this CoverageProvider will try to cover - # every identifier in the database. - INPUT_IDENTIFIER_TYPES: None | str | object = None - - # This CoverageProvider can work with any Collection that supports - # the OPDS import protocol (e.g. DatabaseTest._default_collection). - PROTOCOL: str | None = OPDSAPI.label() - - -class InstrumentedCoverageProvider(MockCoverageProvider, IdentifierCoverageProvider): - """A CoverageProvider that keeps track of every item it tried - to cover. - """ - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.attempts = [] - self.finalize_batch_called = False - - def process_item(self, item): - self.attempts.append(item) - return item - - def finalize_batch(self): - self.finalize_batch_called = True - - -class AlwaysSuccessfulCollectionCoverageProvider( - MockCoverageProvider, CollectionCoverageProvider -): - """A CollectionCoverageProvider that does nothing and always succeeds.""" - - SERVICE_NAME = "Always successful (collection)" - - def process_item(self, item): - return item - - -class AlwaysSuccessfulCoverageProvider(InstrumentedCoverageProvider): - """A CoverageProvider that does nothing and always succeeds.""" - - SERVICE_NAME = "Always successful" - - -class AlwaysSuccessfulBibliographicCoverageProvider( - MockCoverageProvider, BibliographicCoverageProvider -): - """A BibliographicCoverageProvider that does nothing and is always - successful. - - Note that this only works if you've put a working Edition and - LicensePool in place beforehand. Otherwise the process will fail - during handle_success(). - """ - - SERVICE_NAME = "Always successful (bibliographic)" - - def process_item(self, identifier): - return identifier - - -class NeverSuccessfulCoverageProvider(InstrumentedCoverageProvider): - """A CoverageProvider that does nothing and always fails.""" - - SERVICE_NAME = "Never successful" - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.transient = kwargs.get("transient") or False - - def process_item(self, item): - self.attempts.append(item) - return self.failure(item, "What did you expect?", self.transient) - - -class NeverSuccessfulBibliographicCoverageProvider( - MockCoverageProvider, BibliographicCoverageProvider -): - """Simulates a BibliographicCoverageProvider that's never successful.""" - - SERVICE_NAME = "Never successful (bibliographic)" - - def process_item(self, identifier): - return self.failure(identifier, "Bitter failure", transient=True) - - -class TransientFailureCoverageProvider(InstrumentedCoverageProvider): - SERVICE_NAME = "Never successful (transient)" - - def process_item(self, item): - self.attempts.append(item) - return self.failure(item, "Oops!", True) - - -class TaskIgnoringCoverageProvider(InstrumentedCoverageProvider): - """A coverage provider that ignores all work given to it.""" - - SERVICE_NAME = "I ignore all work." - - def process_batch(self, batch): - return [] - - class MockRepresentationHTTPClient: def __init__(self) -> None: self.responses: list[HttpResponseTuple] = []