import datetime
import dateutil
import json
import pytz
import re
import requests
import flask
import urllib.parse
from flask_babel import lazy_gettext as _
from sqlalchemy.orm import contains_eager
from .circulation import (
DeliveryMechanismInfo,
LoanInfo,
HoldInfo,
FulfillmentInfo,
BaseCirculationAPI,
)
from .selftest import (
HasSelfTests,
SelfTestResult,
)
from core.overdrive import (
OverdriveAPI as BaseOverdriveAPI,
OverdriveRepresentationExtractor,
OverdriveBibliographicCoverageProvider,
MockOverdriveAPI as BaseMockOverdriveAPI,
)
from core.model import (
CirculationEvent,
Collection,
Credential,
DataSource,
DeliveryMechanism,
Edition,
ExternalIntegration,
Hold,
Identifier,
LicensePool,
Loan,
MediaTypes,
Representation,
Session,
)
from core.monitor import (
CollectionMonitor,
IdentifierSweepMonitor,
TimelineMonitor,
)
from core.util.datetime_helpers import strptime_utc
from core.util.http import HTTP
from core.metadata_layer import ReplacementPolicy
from core.scripts import Script
from .circulation_exceptions import *
from core.analytics import Analytics
[docs]class OverdriveAPIConstants(object):
# These are not real Overdrive formats; we use them internally so
# we can distinguish between (e.g.) using "audiobook-overdrive"
# to get into Overdrive Read, and using it to get a link to a
# manifest file.
MANIFEST_INTERNAL_FORMATS = set(
['audiobook-overdrive-manifest', 'ebook-overdrive-manifest']
)
# These formats can be delivered either as manifest files or as
# links to websites that stream the content.
STREAMING_FORMATS = [
'ebook-overdrive',
'audiobook-overdrive',
]
[docs]class OverdriveAPI(BaseOverdriveAPI, BaseCirculationAPI, HasSelfTests, OverdriveAPIConstants):
NAME = ExternalIntegration.OVERDRIVE
DESCRIPTION = _("Integrate an Overdrive collection. For an Overdrive Advantage collection, select the consortium's Overdrive collection as the parent.")
SETTINGS = [
{ "key": Collection.EXTERNAL_ACCOUNT_ID_KEY, "label": _("Library ID"), "required": True },
{ "key": BaseOverdriveAPI.WEBSITE_ID, "label": _("Website ID"), "required": True },
{ "key": ExternalIntegration.USERNAME, "label": _("Client Key"), "required": True },
{ "key": ExternalIntegration.PASSWORD, "label": _("Client Secret"), "required": True },
{
"key": BaseOverdriveAPI.SERVER_NICKNAME,
"label": _("Server family"),
"description": _("Unless you hear otherwise from Overdrive, your integration should use their production servers."),
"type": "select",
"options": [
dict(
label=_("Production"),
key=BaseOverdriveAPI.PRODUCTION_SERVERS
),
dict(
label=_("Testing"),
key=BaseOverdriveAPI.TESTING_SERVERS,
)
],
"default": BaseOverdriveAPI.PRODUCTION_SERVERS,
},
] + BaseCirculationAPI.SETTINGS
LIBRARY_SETTINGS = BaseCirculationAPI.LIBRARY_SETTINGS + [
{ "key": BaseOverdriveAPI.ILS_NAME_KEY, "label": _("ILS Name"),
"default": BaseOverdriveAPI.ILS_NAME_DEFAULT,
"description": _("When multiple libraries share an Overdrive account, Overdrive uses a setting called 'ILS Name' to determine which ILS to check when validating a given patron."),
},
BaseCirculationAPI.DEFAULT_LOAN_DURATION_SETTING
]
# An Overdrive Advantage collection inherits everything except the library id
# from its parent.
CHILD_SETTINGS = [
{ "key": Collection.EXTERNAL_ACCOUNT_ID_KEY, "label": _("Library ID"), "required": True },
]
SET_DELIVERY_MECHANISM_AT = BaseCirculationAPI.FULFILL_STEP
# Create a lookup table between common DeliveryMechanism identifiers
# and Overdrive format types.
epub = Representation.EPUB_MEDIA_TYPE
pdf = Representation.PDF_MEDIA_TYPE
adobe_drm = DeliveryMechanism.ADOBE_DRM
no_drm = DeliveryMechanism.NO_DRM
streaming_drm = DeliveryMechanism.STREAMING_DRM
streaming_text = DeliveryMechanism.STREAMING_TEXT_CONTENT_TYPE
streaming_audio = DeliveryMechanism.STREAMING_AUDIO_CONTENT_TYPE
overdrive_audiobook_manifest = MediaTypes.OVERDRIVE_AUDIOBOOK_MANIFEST_MEDIA_TYPE
libby_drm = DeliveryMechanism.LIBBY_DRM
# When a request comes in for a given DeliveryMechanism, what
# do we tell Overdrive?
delivery_mechanism_to_internal_format = {
(epub, no_drm): 'ebook-epub-open',
(epub, adobe_drm): 'ebook-epub-adobe',
(pdf, no_drm): 'ebook-pdf-open',
(pdf, adobe_drm): 'ebook-pdf-adobe',
(streaming_text, streaming_drm): 'ebook-overdrive',
(streaming_audio, streaming_drm): 'audiobook-overdrive',
(overdrive_audiobook_manifest, libby_drm): 'audiobook-overdrive-manifest'
}
# Once you choose a non-streaming format you're locked into it and can't
# use other formats.
LOCK_IN_FORMATS = [
x for x in BaseOverdriveAPI.FORMATS
if x not in OverdriveAPIConstants.STREAMING_FORMATS
and x not in OverdriveAPIConstants.MANIFEST_INTERNAL_FORMATS
]
# TODO: This is a terrible choice but this URL should never be
# displayed to a patron, so it doesn't matter much.
DEFAULT_ERROR_URL = "http://librarysimplified.org/"
# Map Overdrive's error messages to standard circulation manager
# exceptions.
ERROR_MESSAGE_TO_EXCEPTION = {
"PatronHasExceededCheckoutLimit": PatronLoanLimitReached,
"PatronHasExceededCheckoutLimit_ForCPC": PatronLoanLimitReached,
}
def __init__(self, _db, collection):
super(OverdriveAPI, self).__init__(_db, collection)
self.overdrive_bibliographic_coverage_provider = (
OverdriveBibliographicCoverageProvider(
collection, api_class=self
)
)
[docs] def external_integration(self, _db):
return self.collection.external_integration
def _run_self_tests(self, _db):
result = self.run_test(
"Checking global Client Authentication privileges",
self.check_creds, force_refresh=True
)
yield result
if not result.success:
# There is no point in running the other tests if we
# can't even get a token.
return
def _count_advantage():
"""Count the Overdrive Advantage accounts"""
accounts = list(self.get_advantage_accounts())
return "Found %d Overdrive Advantage account(s)." % len(accounts)
yield self.run_test(
"Looking up Overdrive Advantage accounts",
_count_advantage
)
def _count_books():
"""Count the titles in the collection."""
url = self._all_products_link
status, headers, body = self.get(url, {})
body = json.loads(body)
return "%d item(s) in collection" % body['totalItems']
yield self.run_test(
"Counting size of collection", _count_books
)
default_patrons = []
for result in self.default_patrons(self.collection):
if isinstance(result, SelfTestResult):
yield result
continue
library, patron, pin = result
task = "Checking Patron Authentication privileges, using test patron for library %s" % library.name
yield self.run_test(
task, self.get_patron_credential, patron, pin
)
[docs] def patron_request(self, patron, pin, url, extra_headers={}, data=None,
exception_on_401=False, method=None):
"""Make an HTTP request on behalf of a patron.
The results are never cached.
"""
patron_credential = self.get_patron_credential(patron, pin)
headers = dict(Authorization="Bearer %s" % patron_credential.credential)
headers.update(extra_headers)
if method and method.lower() in ('get', 'post', 'put', 'delete'):
method = method.lower()
else:
if data:
method = 'post'
else:
method = 'get'
url = self.endpoint(url)
response = HTTP.request_with_timeout(
method, url, headers=headers, data=data
)
if response.status_code == 401:
if exception_on_401:
# This is our second try. Give up.
raise IntegrationException(
"Something's wrong with the patron OAuth Bearer Token!"
)
else:
# Refresh the token and try again.
self.refresh_patron_access_token(
patron_credential, patron, pin)
return self.patron_request(
patron, pin, url, extra_headers, data, True)
else:
# This is commented out because it may expose patron
# information.
#
# self.log.debug("%s: %s", url, response.status_code)
return response
[docs] def get_patron_credential(self, patron, pin):
"""Create an OAuth token for the given patron."""
def refresh(credential):
return self.refresh_patron_access_token(
credential, patron, pin)
return Credential.lookup(
self._db, DataSource.OVERDRIVE, "OAuth Token", patron, refresh,
collection=self.collection
)
[docs] def scope_string(self, library):
"""Create the Overdrive scope string for the given library.
This is used when setting up Patron Authentication, and when
generating the X-Overdrive-Scope header used by SimplyE to set up
its own Patron Authentication.
"""
return "websiteid:%s authorizationname:%s" % (
self.website_id.decode("utf-8"), self.ils_name(library)
)
[docs] def refresh_patron_access_token(self, credential, patron, pin):
"""Request an OAuth bearer token that allows us to act on
behalf of a specific patron.
Documentation: https://developer.overdrive.com/apis/patron-auth
"""
payload = dict(
grant_type="password",
username=patron.authorization_identifier,
scope=self.scope_string(patron.library)
)
if pin:
# A PIN was provided.
payload['password'] = pin
else:
# No PIN was provided. Depending on the library,
# this might be fine. If it's not fine, Overdrive will
# refuse to issue a token.
payload['password_required'] = 'false'
payload['password'] = '[ignore]'
response = self.token_post(self.PATRON_TOKEN_ENDPOINT, payload)
if response.status_code == 200:
self._update_credential(credential, response.json())
elif response.status_code == 400:
response = response.json()
message = response['error']
error = response.get('error_description')
if error:
message += '/' + error
diagnostic = None
debug = message
if error == 'Requested record not found':
debug = "The patron failed Overdrive's cross-check against the library's ILS."
raise PatronAuthorizationFailedException(message, debug)
return credential
[docs] def checkout(self, patron, pin, licensepool, internal_format):
"""Check out a book on behalf of a patron.
:param patron: a Patron object for the patron who wants
to check out the book.
:param pin: The patron's alleged password.
:param licensepool: Identifier of the book to be checked out is
attached to this licensepool.
:param internal_format: Represents the patron's desired book format.
:return: a LoanInfo object.
"""
identifier = licensepool.identifier
overdrive_id=identifier.identifier
headers = {"Content-Type": "application/json"}
payload = dict(fields=[dict(name="reserveId", value=overdrive_id)])
payload = json.dumps(payload)
response = self.patron_request(
patron, pin, self.CHECKOUTS_ENDPOINT, extra_headers=headers,
data=payload
)
data = response.json()
if response.status_code == 400:
return self._process_checkout_error(patron, pin, licensepool, data)
else:
# Try to extract the expiration date from the response.
expires = self.extract_expiration_date(data)
# Create the loan info.
loan = LoanInfo(
licensepool.collection,
licensepool.data_source.name,
licensepool.identifier.type,
licensepool.identifier.identifier,
None,
expires,
None,
)
return loan
def _process_checkout_error(self, patron, pin, licensepool, error):
"""Handle an error received by the API checkout endpoint.
:param patron: The Patron who tried to check out the book.
:param pin: The Patron's PIN; used in case follow-up
API requests are necessary.
:param licensepool: LicensePool for the book that was to be borrowed.
:param error: A dictionary representing the error response, parsed as JSON.
"""
code = "Unknown Error"
identifier = licensepool.identifier
if isinstance(error, dict):
code = error.get('errorCode', code)
if code == 'NoCopiesAvailable':
# Clearly our info is out of date.
self.update_licensepool(identifier.identifier)
raise NoAvailableCopies()
if code == 'TitleAlreadyCheckedOut':
# Client should have used a fulfill link instead, but
# we can handle it.
#
# NOTE: It's very unlikely this will happen, but it could
# happen if the patron borrows a book through Libby and
# then immediately borrows the same book through SimplyE.
loan = self.get_loan(patron, pin, identifier.identifier)
expires = self.extract_expiration_date(loan)
return LoanInfo(
licensepool.collection,
licensepool.data_source.name,
identifier.type,
identifier.identifier,
None,
expires,
None
)
if code in self.ERROR_MESSAGE_TO_EXCEPTION:
exc_class = self.ERROR_MESSAGE_TO_EXCEPTION[code]
raise exc_class()
# All-purpose fallback
raise CannotLoan(code)
[docs] def checkin(self, patron, pin, licensepool):
# Get the loan for this patron to see whether or not they
# have a delivery mechanism recorded.
loan = None
loans = [l for l in patron.loans if l.license_pool == licensepool]
if loans:
loan = loans[0]
if (loan and loan.fulfillment
and loan.fulfillment.delivery_mechanism
and loan.fulfillment.delivery_mechanism.drm_scheme
== DeliveryMechanism.NO_DRM):
# This patron fulfilled this loan without DRM. That means we
# should be able to find a loanEarlyReturnURL and hit it.
if self.perform_early_return(patron, pin, loan):
# No need for the fallback strategy.
return
# Our fallback strategy is to DELETE the checkout endpoint.
# We do this if no loan can be found, no delivery mechanism is
# recorded, the delivery mechanism uses DRM, we are unable to
# locate the return URL, or we encounter a problem using the
# return URL.
#
# The only case where this is likely to work is when the
# loan exists but has not been locked to a delivery mechanism.
overdrive_id = licensepool.identifier.identifier
url = self.endpoint(self.CHECKOUT_ENDPOINT, overdrive_id=overdrive_id)
return self.patron_request(patron, pin, url, method='DELETE')
@classmethod
def _extract_early_return_url(cls, location):
"""Extract an early return URL from the URL Overdrive sends to
fulfill a non-DRMed book.
:param location: A URL found in a Location header.
"""
if not location:
return None
parsed = urllib.parse.urlparse(location)
query = urllib.parse.parse_qs(parsed.query)
urls = query.get('loanEarlyReturnUrl')
if urls:
return urls[0]
error_to_exception = {
"TitleNotCheckedOut" : NoActiveLoan,
}
[docs] def raise_exception_on_error(self, data, custom_error_to_exception={}):
if not 'errorCode' in data:
return
error = data['errorCode']
message = data.get('message') or ''
for d in custom_error_to_exception, self.error_to_exception:
if error in d:
raise d[error](message)
[docs] def get_loan(self, patron, pin, overdrive_id):
url = self.CHECKOUTS_ENDPOINT + "/" + overdrive_id.upper()
data = self.patron_request(patron, pin, url).json()
self.raise_exception_on_error(data)
return data
[docs] def get_hold(self, patron, pin, overdrive_id):
url = self.endpoint(
self.HOLD_ENDPOINT, product_id=overdrive_id.upper()
)
data = self.patron_request(patron, pin, url).json()
self.raise_exception_on_error(data)
return data
[docs] def get_loans(self, patron, pin):
"""Get a JSON structure describing all of a patron's outstanding
loans."""
data = self.patron_request(patron, pin, self.CHECKOUTS_ENDPOINT).json()
self.raise_exception_on_error(data)
return data
[docs] def fulfill(self, patron, pin, licensepool, internal_format, **kwargs):
"""Get the actual resource file to the patron.
:param kwargs: A container for arguments to fulfill()
which are not relevant to this vendor.
:return: a FulfillmentInfo object.
"""
try:
result = self.get_fulfillment_link(
patron, pin, licensepool.identifier.identifier, internal_format
)
if isinstance(result, FulfillmentInfo):
# The fulfillment process was short-circuited, probably
# by the creation of an OverdriveManifestFulfillmentInfo.
return result
url, media_type = result
if internal_format in self.STREAMING_FORMATS:
media_type += DeliveryMechanism.STREAMING_PROFILE
except FormatNotAvailable as e:
# It's possible the available formats for this book have changed and we
# have an inaccurate delivery mechanism. Try to update the formats, but
# reraise the error regardless.
self.log.info("Overdrive id %s was not available as %s, getting updated formats" % (licensepool.identifier.identifier, internal_format))
try:
self.update_formats(licensepool)
except Exception as e2:
self.log.error("Could not update formats for Overdrive ID %s" % licensepool.identifier.identifier)
raise e
return FulfillmentInfo(
licensepool.collection,
licensepool.data_source.name,
licensepool.identifier.type,
licensepool.identifier.identifier,
content_link=url,
content_type=media_type,
content=None,
content_expires=None
)
[docs] def get_fulfillment_link(self, patron, pin, overdrive_id, format_type):
"""Get the link to the ACSM or manifest for an existing loan.
"""
loan = self.get_loan(patron, pin, overdrive_id)
if not loan:
raise NoActiveLoan("Could not find active loan for %s" % overdrive_id)
download_link = None
if (not loan.get('isFormatLockedIn')
and format_type in self.LOCK_IN_FORMATS):
# The format is not locked in. Lock it in.
# This will happen the first time someone tries to fulfill
# a loan with a lock-in format (basically Adobe-gated formats)
response = self.lock_in_format(
patron, pin, overdrive_id, format_type)
if response.status_code not in (201, 200):
if response.status_code == 400:
message = response.json().get("message")
if message == "The selected format may not be available for this title.":
raise FormatNotAvailable("This book is not available in the format you requested.")
else:
raise CannotFulfill("Could not lock in format %s" % format_type)
response = response.json()
try:
download_link = self.extract_download_link(
response, self.DEFAULT_ERROR_URL)
except IOError as e:
# Get the loan fresh and see if that solves the problem.
loan = self.get_loan(patron, pin, overdrive_id)
# TODO: Verify that the asked-for format type is the same as the
# one in the loan.
if format_type and not download_link:
download_link = self.get_download_link(
loan, format_type, self.DEFAULT_ERROR_URL
)
if not download_link:
raise CannotFulfill(
"No download link for %s, format %s" % (
overdrive_id, format_type))
if download_link:
if format_type in self.MANIFEST_INTERNAL_FORMATS:
# The client must authenticate using its own
# credentials to fulfill this URL; we can't do it.
scope_string = self.scope_string(patron.library)
return OverdriveManifestFulfillmentInfo(
self.collection, download_link,
overdrive_id, scope_string
)
return self.get_fulfillment_link_from_download_link(
patron, pin, download_link)
raise CannotFulfill("Cannot obtain a download link for patron[%r], overdrive_id[%s], format_type[%s].", patron, overdrive_id, format_type)
[docs] def get_fulfillment_link_from_download_link(self, patron, pin, download_link, fulfill_url=None):
# If this for Overdrive's streaming reader, and the link expires,
# the patron can go back to the circulation manager fulfill url
# again to get a new one.
if not fulfill_url and flask.request:
fulfill_url = flask.request.url
else:
fulfill_url=""
download_link = download_link.replace("{odreadauthurl}", fulfill_url)
download_response = self.patron_request(patron, pin, download_link)
return self.extract_content_link(download_response.json())
@classmethod
def _extract_date(cls, data, field_name):
if not isinstance(data, dict):
return None
if not field_name in data:
return None
try:
return strptime_utc(
data[field_name], cls.TIME_FORMAT
)
except ValueError as e:
# Wrong format
return None
[docs] def get_patron_checkouts(self, patron, pin):
data = self.patron_request(patron, pin, self.CHECKOUTS_ENDPOINT).json()
self.raise_exception_on_error(data)
return data
[docs] def get_patron_holds(self, patron, pin):
data = self.patron_request(patron, pin, self.HOLDS_ENDPOINT).json()
self.raise_exception_on_error(data)
return data
@classmethod
def _pd(cls, d):
"""Stupid method to parse a date.
TIME_FORMAT mentions "Z" for Zulu time, which is the same as
UTC.
"""
if not d:
return d
return strptime_utc(d, cls.TIME_FORMAT)
[docs] def patron_activity(self, patron, pin):
try:
loans = self.get_patron_checkouts(patron, pin)
holds = self.get_patron_holds(patron, pin)
except PatronAuthorizationFailedException as e:
# This frequently happens because Overdrive performs
# checks for blocked or expired accounts upon initial
# authorization, where the circulation manager would let
# the 'authorization' part succeed and block the patron's
# access afterwards.
#
# It's common enough that it's hardly worth mentioning, but it
# could theoretically be the sign of a larger problem.
self.log.info(
"Overdrive authentication failed, assuming no loans.",
exc_info=e
)
loans = {}
holds = {}
for checkout in loans.get('checkouts', []):
loan_info = self.process_checkout_data(checkout, self.collection)
yield loan_info
for hold in holds.get('holds', []):
overdrive_identifier = hold['reserveId'].lower()
start = self._pd(hold.get('holdPlacedDate'))
end = self._pd(hold.get('holdExpires'))
position = hold.get('holdListPosition')
if position is not None:
position = int(position)
if 'checkout' in hold.get('actions', {}):
# This patron needs to decide whether to check the
# book out. By our reckoning, the patron's position is
# 0, not whatever position Overdrive had for them.
position = 0
yield HoldInfo(
self.collection,
DataSource.OVERDRIVE,
Identifier.OVERDRIVE_ID,
overdrive_identifier,
start_date=start,
end_date=end,
hold_position=position
)
[docs] @classmethod
def process_checkout_data(cls, checkout, collection):
"""Convert one checkout from Overdrive's list of checkouts
into a LoanInfo object.
:return: A LoanInfo object if the book can be fulfilled
by the default Library Simplified client, and None otherwise.
"""
overdrive_identifier = checkout['reserveId'].lower()
start = cls._pd(checkout.get('checkoutDate'))
end = cls._pd(checkout.get('expires'))
usable_formats = []
# If a format is already locked in, it will be in formats.
for format in checkout.get('formats', []):
format_type = format.get('formatType')
if format_type in cls.FORMATS:
usable_formats.append(format_type)
# If a format hasn't been selected yet, available formats are in actions.
actions = checkout.get('actions', {})
format_action = actions.get('format', {})
format_fields = format_action.get('fields', [])
for field in format_fields:
if field.get('name', "") == "formatType":
format_options = field.get("options", [])
for format_type in format_options:
if format_type in cls.FORMATS:
usable_formats.append(format_type)
if not usable_formats:
# Either this book is not available in any format readable
# by the default client, or the patron previously chose to
# fulfill it in a format not readable by the default
# client. Either way, we cannot fulfill this loan and we
# shouldn't show it in the list.
return None
locked_to = None
if len(usable_formats) == 1:
# Either the book has been locked into a specific format,
# or only one usable format is available. We don't know
# which case we're looking at, but for our purposes the
# book is locked -- unless, of course, what Overdrive
# considers "one format" corresponds to more than one
# format on our side.
[overdrive_format] = usable_formats
internal_formats = list(
OverdriveRepresentationExtractor.internal_formats(
overdrive_format
)
)
if len(internal_formats) == 1:
[(media_type, drm_scheme)] = internal_formats
# Make it clear that Overdrive will only deliver the content
# in one specific media type.
locked_to = DeliveryMechanismInfo(
content_type=media_type,
drm_scheme=drm_scheme
)
return LoanInfo(
collection,
DataSource.OVERDRIVE,
Identifier.OVERDRIVE_ID,
overdrive_identifier,
start_date=start,
end_date=end,
locked_to=locked_to
)
[docs] def default_notification_email_address(self, patron, pin):
"""Find the email address this patron wants to use for hold
notifications.
:return: The email address Overdrive has on record for
this patron's hold notifications, or None if there is
no such address.
"""
# We're calling the superclass implementation, but we have no
# intention of actually using the result. This is a
# per-library default that trashes all of its input, and
# Overdrive has a better solution.
trash_everything_address = super(
OverdriveAPI, self
).default_notification_email_address(patron, pin)
# Instead, we will ask _Overdrive_ if this patron has a
# preferred email address for notifications.
address = None
response = self.patron_request(
patron, pin, self.PATRON_INFORMATION_ENDPOINT
)
if response.status_code == 200:
data = response.json()
address = data.get('lastHoldEmail')
# Great! Except, it's possible that this address is the
# 'trash everything' address, because we _used_ to send
# that address to Overdrive. If so, ignore it.
if address == trash_everything_address:
address = None
else:
self.log.error(
"Unable to get patron information for %s: %s",
patron.authorization_identifier,
response.content
)
return address
[docs] def place_hold(self, patron, pin, licensepool, notification_email_address):
"""Place a book on hold.
:return: A HoldData object, if a hold was successfully placed,
or the book was already on hold.
:raise: A CirculationException explaining why no hold
could be placed.
"""
if not notification_email_address:
notification_email_address = self.default_notification_email_address(
patron, pin
)
overdrive_id = licensepool.identifier.identifier
form_fields = dict(reserveId=overdrive_id)
if notification_email_address:
form_fields['emailAddress'] = notification_email_address
else:
form_fields['ignoreHoldEmail'] = True
headers, document = self.fill_out_form(**form_fields)
response = self.patron_request(
patron, pin, self.HOLDS_ENDPOINT, headers,
document
)
return self.process_place_hold_response(
response, patron, pin, licensepool
)
[docs] def process_place_hold_response(self, response, patron, pin, licensepool):
"""Process the response to a HOLDS_ENDPOINT request.
:return: A HoldData object, if a hold was successfully placed,
or the book was already on hold.
:raise: A CirculationException explaining why no hold
could be placed.
"""
def make_holdinfo(hold_response):
# Create a HoldInfo object by combining data passed into
# the enclosing method with the data from a hold response
# (either creating a new hold or fetching an existing
# one).
position, start_date = self.extract_data_from_hold_response(
hold_response
)
return HoldInfo(
licensepool.collection,
licensepool.data_source.name,
licensepool.identifier.type,
licensepool.identifier.identifier,
start_date=start_date,
end_date=None,
hold_position=position
)
family = response.status_code // 100
if family == 4:
error = response.json()
if not error or not 'errorCode' in error:
raise CannotHold()
code = error['errorCode']
if code == 'AlreadyOnWaitList':
# The book is already on hold, so this isn't an exceptional
# condition. Refresh the queue info and act as though the
# request was successful.
hold = self.get_hold(
patron, pin, licensepool.identifier.identifier
)
return make_holdinfo(hold)
elif code == 'NotWithinRenewalWindow':
# The patron has this book checked out and cannot yet
# renew their loan.
raise CannotRenew()
elif code == 'PatronExceededHoldLimit':
raise PatronHoldLimitReached()
else:
raise CannotHold(code)
elif family == 2:
# The book was successfuly placed on hold. Return an
# appropriate HoldInfo.
data = response.json()
return make_holdinfo(data)
else:
# Some other problem happened -- we don't know what. It's
# not a 5xx error because the HTTP client would have been
# turned that into a RemoteIntegrationException.
raise CannotHold()
[docs] def release_hold(self, patron, pin, licensepool):
"""Release a patron's hold on a book.
:raises CannotReleaseHold: If there is an error communicating
with Overdrive, or Overdrive refuses to release the hold for
any reason.
"""
url = self.endpoint(
self.HOLD_ENDPOINT,
product_id=licensepool.identifier.identifier
)
response = self.patron_request(patron, pin, url, method='DELETE')
if response.status_code // 100 == 2 or response.status_code == 404:
return True
if not response.content:
raise CannotReleaseHold()
data = response.json()
if not 'errorCode' in data:
raise CannotReleaseHold()
if data['errorCode'] == 'PatronDoesntHaveTitleOnHold':
# There was never a hold to begin with, so we're fine.
return True
raise CannotReleaseHold(response.content)
[docs] def circulation_lookup(self, book):
if isinstance(book, str):
book_id = book
circulation_link = self.endpoint(
self.AVAILABILITY_ENDPOINT,
collection_token=self.collection_token,
product_id=book_id
)
book = dict(id=book_id)
else:
circulation_link = book['availability_link']
# Make sure we use v2 of the availability API,
# even if Overdrive gave us a link to v1.
circulation_link = self.make_link_safe(circulation_link)
return book, self.get(circulation_link, {})
[docs] def update_licensepool(self, book_id):
"""Update availability information for a single book.
If the book has never been seen before, a new LicensePool
will be created for the book.
The book's LicensePool will be updated with current
circulation information. Bibliographic coverage will be
ensured for the Overdrive Identifier, and a Work will be
created for the LicensePool and set as presentation-ready.
"""
# Retrieve current circulation information about this book
try:
book, (status_code, headers, content) = self.circulation_lookup(
book_id
)
except Exception as e:
status_code = None
self.log.error(
"HTTP exception communicating with Overdrive",
exc_info=e
)
# TODO: If you ask for a book that you know about, and
# Overdrive says the book doesn't exist in the collection,
# then it's appropriate to update an existing
# LicensePool. However we shouldn't be creating a *brand new*
# LicensePool for a book Overdrive says isn't in the
# collection.
if status_code not in (200, 404):
self.log.error(
"Could not get availability for %s: status code %s",
book_id, status_code
)
return None, None, False
if isinstance(content, (bytes, str)):
content = json.loads(content)
book.update(content)
# Update book_id now that we know we have new data.
book_id = book['id']
license_pool, is_new = LicensePool.for_foreign_id(
self._db, DataSource.OVERDRIVE, Identifier.OVERDRIVE_ID, book_id,
collection=self.collection
)
if is_new or not license_pool.work:
# Either this is the first time we've seen this book or its doesn't
# have an associated work. Make sure its identifier has bibliographic coverage.
self.overdrive_bibliographic_coverage_provider.ensure_coverage(
license_pool.identifier,
force=True
)
return self.update_licensepool_with_book_info(
book, license_pool, is_new
)
# Alias for the CirculationAPI interface
[docs] def update_availability(self, licensepool):
return self.update_licensepool(licensepool.identifier.identifier)
def _edition(self, licensepool):
"""Find or create the Edition that would be used to contain
Overdrive metadata for the given LicensePool.
"""
return Edition.for_foreign_id(
self._db, self.source, licensepool.identifier.type,
licensepool.identifier.identifier
)
[docs] def update_licensepool_with_book_info(self, book, license_pool, is_new_pool):
"""Update a book's LicensePool with information from a JSON
representation of its circulation info.
Then, create an Edition and make sure it has bibliographic
coverage. If the new Edition is the only candidate for the
pool's presentation_edition, promote it to presentation
status.
"""
extractor = OverdriveRepresentationExtractor(self)
circulation = extractor.book_info_to_circulation(
book
)
license_pool, circulation_changed = circulation.apply(
self._db, license_pool.collection
)
edition, is_new_edition = self._edition(license_pool)
if is_new_pool:
license_pool.open_access = False
self.log.info("New Overdrive book discovered: %r", edition)
return license_pool, is_new_pool, circulation_changed
[docs] @classmethod
def get_download_link(self, checkout_response, format_type, error_url):
"""Extract a download link from the given response.
:param checkout_response: A JSON document describing a checkout-type
response from the Overdrive API.
:param format_type: The internal (Overdrive-facing) format type
that should be retrieved. 'x-manifest' format types are treated
as a variant of the 'x' format type -- Overdrive doesn't recognise
'x-manifest' and uses 'x' for delivery of both streaming content
and manifests.
:param error_url: Value to interpolate for the {errorpageurl}
URI template value. This is ignored if you're fetching a manifest;
instead, the 'errorpageurl' variable is removed entirely.
"""
link = None
format = None
available_formats = []
if format_type in self.MANIFEST_INTERNAL_FORMATS:
use_format_type = format_type.replace("-manifest", "")
fetch_manifest = True
else:
use_format_type = format_type
fetch_manifest = False
for f in checkout_response.get('formats', []):
this_type = f['formatType']
available_formats.append(this_type)
if this_type == use_format_type:
format = f
break
if not format:
if any(x in set(available_formats) for x in self.INCOMPATIBLE_PLATFORM_FORMATS):
# The most likely explanation is that the patron
# already had this book delivered to their Kindle.
raise FulfilledOnIncompatiblePlatform(
"It looks like this loan was already fulfilled on another platform, most likely Amazon Kindle. We're not allowed to also send it to you as an EPUB."
)
else:
# We don't know what happened -- most likely our
# format data is bad.
format_list = ", ".join(available_formats)
msg = "Could not find specified format %s. Available formats: %s"
raise NoAcceptableFormat(
msg % (use_format_type, ", ".join(available_formats))
)
return self.extract_download_link(format, error_url, fetch_manifest)
[docs] @classmethod
def make_direct_download_link(cls, link):
"""Convert an Overdrive Read or Overdrive Listen link template to a
direct-download link for the manifest.
This means removing any templated arguments for Overdrive Read
authentication URL and error URL; and adding a value for the
'contentfile' argument.
:param link: An Overdrive Read or Overdrive Listen template
link.
"""
# Remove any Overdrive Read authentication URL and error URL.
for argument_name in ('odreadauthurl', 'errorpageurl'):
argument_re = re.compile("%s={%s}&?" % (argument_name, argument_name))
link = argument_re.sub("", link)
# Add the contentfile=true argument.
if '?' not in link:
link += '?contentfile=true'
elif link.endswith('&') or link.endswith('?'):
link += 'contentfile=true'
else:
link += '&contentfile=true'
return link
[docs]class MockOverdriveResponse(object):
def __init__(self, status_code, headers, content):
self.status_code = status_code
self.headers = headers
self.content = content
[docs] def json(self):
return json.loads(self.content)
[docs]class MockOverdriveAPI(BaseMockOverdriveAPI, OverdriveAPI):
library_data = '{"id":1810,"name":"My Public Library (MA)","type":"Library","collectionToken":"1a09d9203","links":{"self":{"href":"http://api.overdrive.com/v1/libraries/1810","type":"application/vnd.overdrive.api+json"},"products":{"href":"http://api.overdrive.com/v1/collections/1a09d9203/products","type":"application/vnd.overdrive.api+json"},"dlrHomepage":{"href":"http://ebooks.nypl.org","type":"text/html"}},"formats":[{"id":"audiobook-wma","name":"OverDrive WMA Audiobook"},{"id":"ebook-pdf-adobe","name":"Adobe PDF eBook"},{"id":"ebook-mediado","name":"MediaDo eBook"},{"id":"ebook-epub-adobe","name":"Adobe EPUB eBook"},{"id":"ebook-kindle","name":"Kindle Book"},{"id":"audiobook-mp3","name":"OverDrive MP3 Audiobook"},{"id":"ebook-pdf-open","name":"Open PDF eBook"},{"id":"ebook-overdrive","name":"OverDrive Read"},{"id":"video-streaming","name":"Streaming Video"},{"id":"ebook-epub-open","name":"Open EPUB eBook"}]}'
token_data = '{"access_token":"foo","token_type":"bearer","expires_in":3600,"scope":"LIB META AVAIL SRCH"}'
collection_token = 'fake token'
[docs] def patron_request(self, patron, pin, *args, **kwargs):
response = self._make_request(*args, **kwargs)
# Modify the record of the request to include the patron information.
original_data = self.requests[-1]
# The last item in the record of the request is keyword arguments.
# Stick this information in there to minimize confusion.
original_data[-1]['_patron'] = patron
original_data[-1]['_pin'] = patron
return response
[docs]class OverdriveCirculationMonitor(CollectionMonitor, TimelineMonitor):
"""Maintain LicensePools for recently changed Overdrive titles. Create
basic Editions for any new LicensePools that show up.
"""
SERVICE_NAME = "Overdrive Circulation Monitor"
PROTOCOL = ExternalIntegration.OVERDRIVE
OVERLAP = datetime.timedelta(minutes=1)
def __init__(self, _db, collection, api_class=OverdriveAPI, analytics_class=Analytics):
"""Constructor."""
super(OverdriveCirculationMonitor, self).__init__(_db, collection)
self.api = api_class(_db, collection)
self.analytics = analytics_class(_db)
[docs] def recently_changed_ids(self, start, cutoff):
return self.api.recently_changed_ids(start, cutoff)
[docs] def catch_up_from(self, start, cutoff, progress):
"""Find Overdrive books that changed recently.
:progress: A TimestampData representing the time previously
covered by this Monitor.
"""
overdrive_data_source = DataSource.lookup(
self._db, DataSource.OVERDRIVE
)
# Ask for changes between the last time covered by the Monitor
# and the current time.
total_books = 0
for book in self.recently_changed_ids(start, cutoff):
total_books += 1
if not total_books % 100:
self.log.info("%s books processed", total_books)
if not book:
continue
license_pool, is_new, is_changed = self.api.update_licensepool(book)
# Log a circulation event for this work.
if is_new:
for library in self.collection.libraries:
self.analytics.collect_event(
library, license_pool, CirculationEvent.DISTRIBUTOR_TITLE_ADD, license_pool.last_checked
)
self._db.commit()
if self.should_stop(start, book, is_changed):
break
progress.achievements = "Books processed: %d." % total_books
[docs]class NewTitlesOverdriveCollectionMonitor(OverdriveCirculationMonitor):
"""Monitor the Overdrive collection for newly added titles.
This catches any new titles that slipped through the cracks of the
RecentOverdriveCollectionMonitor.
"""
SERVICE_NAME = "Overdrive New Title Monitor"
OVERLAP = datetime.timedelta(days=7)
DEFAULT_START_TIME = OverdriveCirculationMonitor.NEVER
[docs] def recently_changed_ids(self, start, cutoff):
"""Ignore the dates and return all IDs."""
return self.api.all_ids()
[docs] def should_stop(self, start, api_description, is_changed):
if not start or start is self.NEVER:
# This monitor has never run before. It should ask about
# every single book.
return False
# We should stop if this book was added before our start time.
date_added = api_description.get('date_added')
if not date_added:
# We don't know when this book was added -- shouldn't happen.
return False
try:
date_added = dateutil.parser.parse(date_added)
except ValueError as e:
# The date format is unparseable -- shouldn't happen.
self.log.error("Got invalid date: %s", date_added)
return False
# The time stored in the database is UTC, but it's stored
# without any time zone information. Add that information so
# we can compare it against the date we got from Overdrive.
start = pytz.utc.localize(start)
self.log.info(
"Date added: %s, start time: %s, result %s",
date_added, start, date_added < start
)
return date_added < start
[docs]class OverdriveCollectionReaper(IdentifierSweepMonitor):
"""Check for books that are in the local collection but have left our
Overdrive collection.
"""
SERVICE_NAME = "Overdrive Collection Reaper"
PROTOCOL = ExternalIntegration.OVERDRIVE
def __init__(self, _db, collection, api_class=OverdriveAPI):
super(OverdriveCollectionReaper, self).__init__(_db, collection)
self.api = api_class(_db, collection)
[docs] def process_item(self, identifier):
self.api.update_licensepool(identifier.identifier)
[docs]class RecentOverdriveCollectionMonitor(OverdriveCirculationMonitor):
"""Monitor recently changed books in the Overdrive collection."""
SERVICE_NAME = "Reverse Chronological Overdrive Collection Monitor"
# Report successful completion upon finding this number of
# consecutive books in the Overdrive results whose LicensePools
# haven't changed since last time. Overdrive results are not in
# strict chronological order, but if you see 100 consecutive books
# that haven't changed, you're probably done.
MAXIMUM_CONSECUTIVE_UNCHANGED_BOOKS=100
def __init__(self, *args, **kwargs):
super(RecentOverdriveCollectionMonitor, self).__init__(*args, **kwargs)
self.consecutive_unchanged_books = 0
[docs] def should_stop(self, start, api_description, is_changed):
if is_changed:
self.consecutive_unchanged_books = 0
else:
self.consecutive_unchanged_books += 1
if (self.consecutive_unchanged_books >=
self.MAXIMUM_CONSECUTIVE_UNCHANGED_BOOKS):
# We're supposed to stop this run after finding a
# run of books that have not changed, and we have
# in fact seen that many consecutive unchanged
# books.
self.log.info("Stopping at %d unchanged books.",
self.consecutive_unchanged_books)
return True
return False
[docs]class OverdriveAdvantageAccountListScript(Script):
[docs] def run(self):
"""Explain every Overdrive collection and, for each one, all of its
Advantage collections.
"""
collections = Collection.by_protocol(
self._db, ExternalIntegration.OVERDRIVE
)
for collection in collections:
self.explain_main_collection(collection)
print()
[docs] def explain_main_collection(self, collection):
"""Explain an Overdrive collection and all of its Advantage
collections.
"""
api = OverdriveAPI(self._db, collection)
print("Main Overdrive collection: %s" % collection.name)
print("\n".join(collection.explain()))
print("A few of the titles in the main collection:")
for i, book in enumerate(api.all_ids()):
print("", book['title'])
if i > 10:
break
advantage_accounts = list(api.get_advantage_accounts())
print("%d associated Overdrive Advantage account(s)." % len(
advantage_accounts
))
for advantage_collection in advantage_accounts:
self.explain_advantage_collection(advantage_collection)
print()
[docs] def explain_advantage_collection(self, collection):
"""Explain a single Overdrive Advantage collection."""
parent_collection, child = collection.to_collection(self._db)
print(" Overdrive Advantage collection: %s" % child.name)
print(" " + ("\n ".join(child.explain())))
print(" A few of the titles in this Advantage collection:")
child_api = OverdriveAPI(self._db, child)
for i, book in enumerate(child_api.all_ids()):
print(" ", book['title'])
if i > 10:
break
[docs]class OverdriveManifestFulfillmentInfo(FulfillmentInfo):
def __init__(self, collection, content_link, overdrive_identifier,
scope_string):
"""Constructor.
Most of the arguments to the superconstructor can be assumed,
and none of them matter all that much, since this class
overrides the normal process by which a FulfillmentInfo becomes
a Flask response.
"""
super(OverdriveManifestFulfillmentInfo, self).__init__(
collection=collection,
data_source_name=DataSource.OVERDRIVE,
identifier_type=Identifier.OVERDRIVE_ID,
identifier=overdrive_identifier,
content_link=content_link,
content_type=None,
content=None,
content_expires=None,
)
self.scope_string = scope_string
@property
def as_response(self):
headers = {
"Location": self.content_link,
"X-Overdrive-Scope": self.scope_string,
"Content-Type": self.content_type or 'text/plain',
}
return flask.Response("", 302, headers)