2020-01-02 16:35:18 +00:00
|
|
|
|
import urllib
|
|
|
|
|
|
|
2019-09-05 17:36:02 +01:00
|
|
|
|
from flask import current_app
|
2021-03-10 13:55:06 +00:00
|
|
|
|
from notifications_utils.s3 import S3ObjectNotFound
|
|
|
|
|
|
from notifications_utils.s3 import s3download as utils_s3download
|
2018-04-25 12:19:12 +01:00
|
|
|
|
from sqlalchemy.orm.exc import NoResultFound
|
|
|
|
|
|
|
2018-11-30 16:35:00 +00:00
|
|
|
|
from app import create_random_identifier
|
2017-07-19 13:50:29 +01:00
|
|
|
|
from app.config import QueueNames
|
Don't error sending a letter that's sent already
Fixes this error (in Admin):
File "/home/vcap/app/app/notify_client/notification_api_client.py", line 74, in send_precompiled_letter
return self.post(url='/service/{}/send-pdf-letter'.format(service_id), data=data)
File "/home/vcap/app/app/notify_client/__init__.py", line 59, in post
return super().post(*args, **kwargs)
File "/home/vcap/deps/0/python/lib/python3.9/site-packages/notifications_python_client/base.py", line 48, in post
return self.request("POST", url, data=data)
File "/home/vcap/deps/0/python/lib/python3.9/site-packages/notifications_python_client/base.py", line 64, in request
response = self._perform_request(method, url, kwargs)
File "/home/vcap/deps/0/python/lib/python3.9/site-packages/notifications_python_client/base.py", line 118, in _perform_request
raise api_error
notifications_python_client.errors.HTTPError: 500 - Internal server error
Due to this error (in API):
File "/home/vcap/app/app/service/send_notification.py", line 178, in send_pdf_letter_notification
raise e
File "/home/vcap/app/app/service/send_notification.py", line 173, in send_pdf_letter_notification
letter = utils_s3download(current_app.config['TRANSIENT_UPLOADED_LETTERS'], file_location)
File "/home/vcap/deps/0/python/lib/python3.9/site-packages/notifications_utils/s3.py", line 53, in s3download
raise S3ObjectNotFound(error.response, error.operation_name)
notifications_utils.s3.S3ObjectNotFound: An error occurred (NoSuchKey) when calling the GetObject operation: The specified key does not exist.
I checked the DB to verify the letter does actually exist i.e. it
is an instance of the problem we're fixing here.
2022-04-08 17:20:44 +01:00
|
|
|
|
from app.dao.notifications_dao import (
|
|
|
|
|
|
_update_notification_status,
|
|
|
|
|
|
get_notification_by_id,
|
|
|
|
|
|
)
|
2017-11-25 11:31:36 +00:00
|
|
|
|
from app.dao.service_email_reply_to_dao import dao_get_reply_to_by_id
|
|
|
|
|
|
from app.dao.service_sms_sender_dao import dao_get_service_sms_senders_by_id
|
2017-06-13 17:33:04 +01:00
|
|
|
|
from app.dao.services_dao import dao_fetch_service_by_id
|
2021-03-10 13:55:06 +00:00
|
|
|
|
from app.dao.templates_dao import (
|
|
|
|
|
|
dao_get_template_by_id_and_service_id,
|
|
|
|
|
|
get_precompiled_letter_template,
|
|
|
|
|
|
)
|
2017-06-16 16:30:03 +01:00
|
|
|
|
from app.dao.users_dao import get_user_by_id
|
2019-09-05 17:36:02 +01:00
|
|
|
|
from app.letters.utils import (
|
Relax lookup of letter PDFs in S3 buckets
Previously we generated the filename we expected a letter PDF to be
stored at in S3, and used that to retrieve it. However, the generated
filename can change over the course of a notification's lifetime e.g.
if the service changes from crown ('.C.') to non-crown ('.N.').
The prefix of the filename is stable: it's based on properties of the
notification - reference and creation - that don't change. This commit
changes the way we interact with letter PDFs in S3:
- Uploading uses the original method to generate the full file name.
The method is renamed to 'generate_' to distinguish it from the new one.
- Downloading uses a new 'find_' method to get the filename using just
its prefix, which makes it agnostic to changes in the filename suffix.
Making this change helps to decouple our code from the requirements DVLA
have on the filenames. While it means more traffic to S3, we rely on S3
in any case to download the files. From experience, we know S3 is highly
reliable and performant, so don't anticipate any issues.
In the tests we favour using moto to mock S3, so that the behaviour is
realistic. There are a couple of places where we just mock the method,
since what it returns isn't important for the test.
Note that, since the new method requires a notification object, we need
to change a query in one place, the columns of which were only selected
to appease the original method to generate a filename.
2021-03-08 15:23:37 +00:00
|
|
|
|
generate_letter_pdf_filename,
|
2019-10-16 13:35:30 +01:00
|
|
|
|
get_billable_units_for_letter_page_count,
|
2019-09-05 17:36:02 +01:00
|
|
|
|
get_page_count,
|
|
|
|
|
|
move_uploaded_pdf_to_letters_bucket,
|
|
|
|
|
|
)
|
2021-03-10 13:55:06 +00:00
|
|
|
|
from app.models import (
|
|
|
|
|
|
EMAIL_TYPE,
|
|
|
|
|
|
KEY_TYPE_NORMAL,
|
|
|
|
|
|
LETTER_TYPE,
|
|
|
|
|
|
NOTIFICATION_DELIVERED,
|
|
|
|
|
|
PRIORITY,
|
|
|
|
|
|
SMS_TYPE,
|
|
|
|
|
|
)
|
|
|
|
|
|
from app.notifications.process_notifications import (
|
|
|
|
|
|
persist_notification,
|
|
|
|
|
|
send_notification_to_queue,
|
|
|
|
|
|
)
|
|
|
|
|
|
from app.notifications.validators import (
|
|
|
|
|
|
check_service_has_permission,
|
|
|
|
|
|
check_service_over_daily_message_limit,
|
|
|
|
|
|
validate_address,
|
|
|
|
|
|
validate_and_format_recipient,
|
|
|
|
|
|
validate_template,
|
|
|
|
|
|
)
|
2017-06-16 16:30:03 +01:00
|
|
|
|
from app.v2.errors import BadRequestError
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def validate_created_by(service, created_by_id):
|
|
|
|
|
|
user = get_user_by_id(created_by_id)
|
|
|
|
|
|
if service not in user.services:
|
|
|
|
|
|
message = 'Can’t create notification - {} is not part of the "{}" service'.format(
|
|
|
|
|
|
user.name,
|
|
|
|
|
|
service.name
|
|
|
|
|
|
)
|
|
|
|
|
|
raise BadRequestError(message=message)
|
2017-06-13 17:33:04 +01:00
|
|
|
|
|
|
|
|
|
|
|
2018-11-30 16:35:00 +00:00
|
|
|
|
def create_one_off_reference(template_type):
|
2018-12-03 11:33:59 +00:00
|
|
|
|
if template_type == LETTER_TYPE:
|
|
|
|
|
|
return create_random_identifier()
|
|
|
|
|
|
return None
|
2018-11-30 16:35:00 +00:00
|
|
|
|
|
|
|
|
|
|
|
2017-06-13 17:33:04 +01:00
|
|
|
|
def send_one_off_notification(service_id, post_data):
|
|
|
|
|
|
service = dao_fetch_service_by_id(service_id)
|
|
|
|
|
|
template = dao_get_template_by_id_and_service_id(
|
|
|
|
|
|
template_id=post_data['template_id'],
|
|
|
|
|
|
service_id=service_id
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
personalisation = post_data.get('personalisation', None)
|
|
|
|
|
|
|
2017-06-15 13:40:38 +01:00
|
|
|
|
validate_template(template.id, personalisation, service, template.template_type)
|
2017-06-13 17:33:04 +01:00
|
|
|
|
|
|
|
|
|
|
check_service_over_daily_message_limit(KEY_TYPE_NORMAL, service)
|
|
|
|
|
|
|
|
|
|
|
|
validate_and_format_recipient(
|
|
|
|
|
|
send_to=post_data['to'],
|
|
|
|
|
|
key_type=KEY_TYPE_NORMAL,
|
|
|
|
|
|
service=service,
|
2018-01-17 15:20:04 +00:00
|
|
|
|
notification_type=template.template_type,
|
2020-07-28 10:19:46 +01:00
|
|
|
|
allow_guest_list_recipients=False,
|
2017-06-13 17:33:04 +01:00
|
|
|
|
)
|
2020-07-29 14:52:18 +01:00
|
|
|
|
postage = None
|
2021-03-19 16:42:55 +00:00
|
|
|
|
client_reference = None
|
2020-07-29 14:52:18 +01:00
|
|
|
|
if template.template_type == LETTER_TYPE:
|
|
|
|
|
|
# Validate address and set postage to europe|rest-of-world if international letter,
|
|
|
|
|
|
# otherwise persist_notification with use template postage
|
|
|
|
|
|
postage = validate_address(service, personalisation)
|
2020-08-04 15:41:42 +01:00
|
|
|
|
if not postage:
|
|
|
|
|
|
postage = template.postage
|
2021-03-19 16:42:55 +00:00
|
|
|
|
from app.utils import get_reference_from_personalisation
|
|
|
|
|
|
client_reference = get_reference_from_personalisation(personalisation)
|
|
|
|
|
|
|
2017-06-16 16:30:03 +01:00
|
|
|
|
validate_created_by(service, post_data['created_by'])
|
|
|
|
|
|
|
2017-11-25 11:31:36 +00:00
|
|
|
|
sender_id = post_data.get('sender_id', None)
|
2017-12-15 17:13:55 +00:00
|
|
|
|
reply_to = get_reply_to_text(
|
|
|
|
|
|
notification_type=template.template_type,
|
|
|
|
|
|
sender_id=sender_id,
|
|
|
|
|
|
service=service,
|
|
|
|
|
|
template=template
|
|
|
|
|
|
)
|
2017-06-13 17:33:04 +01:00
|
|
|
|
notification = persist_notification(
|
|
|
|
|
|
template_id=template.id,
|
|
|
|
|
|
template_version=template.version,
|
|
|
|
|
|
recipient=post_data['to'],
|
|
|
|
|
|
service=service,
|
|
|
|
|
|
personalisation=personalisation,
|
|
|
|
|
|
notification_type=template.template_type,
|
|
|
|
|
|
api_key_id=None,
|
2017-06-16 16:30:03 +01:00
|
|
|
|
key_type=KEY_TYPE_NORMAL,
|
2017-11-25 11:31:36 +00:00
|
|
|
|
created_by_id=post_data['created_by'],
|
2018-11-30 16:35:00 +00:00
|
|
|
|
reply_to_text=reply_to,
|
|
|
|
|
|
reference=create_one_off_reference(template.template_type),
|
2021-03-19 16:42:55 +00:00
|
|
|
|
postage=postage,
|
|
|
|
|
|
client_reference=client_reference
|
2017-06-13 17:33:04 +01:00
|
|
|
|
)
|
|
|
|
|
|
|
2018-03-01 10:17:59 +00:00
|
|
|
|
queue_name = QueueNames.PRIORITY if template.process_type == PRIORITY else None
|
2018-10-31 14:30:46 +00:00
|
|
|
|
|
|
|
|
|
|
if template.template_type == LETTER_TYPE and service.research_mode:
|
|
|
|
|
|
_update_notification_status(
|
|
|
|
|
|
notification,
|
|
|
|
|
|
NOTIFICATION_DELIVERED,
|
|
|
|
|
|
)
|
|
|
|
|
|
else:
|
|
|
|
|
|
send_notification_to_queue(
|
|
|
|
|
|
notification=notification,
|
|
|
|
|
|
research_mode=service.research_mode,
|
|
|
|
|
|
queue=queue_name,
|
|
|
|
|
|
)
|
2017-06-13 17:33:04 +01:00
|
|
|
|
|
|
|
|
|
|
return {'id': str(notification.id)}
|
2017-11-25 11:31:36 +00:00
|
|
|
|
|
|
|
|
|
|
|
2017-12-15 17:13:55 +00:00
|
|
|
|
def get_reply_to_text(notification_type, sender_id, service, template):
|
2017-11-25 11:31:36 +00:00
|
|
|
|
reply_to = None
|
2017-12-15 17:13:55 +00:00
|
|
|
|
if sender_id:
|
2018-04-25 14:23:00 +01:00
|
|
|
|
try:
|
|
|
|
|
|
if notification_type == EMAIL_TYPE:
|
2018-04-25 12:19:12 +01:00
|
|
|
|
message = 'Reply to email address not found'
|
2018-04-25 14:23:00 +01:00
|
|
|
|
reply_to = dao_get_reply_to_by_id(service.id, sender_id).email_address
|
|
|
|
|
|
elif notification_type == SMS_TYPE:
|
|
|
|
|
|
message = 'SMS sender not found'
|
|
|
|
|
|
reply_to = dao_get_service_sms_senders_by_id(service.id, sender_id).get_reply_to_text()
|
|
|
|
|
|
except NoResultFound:
|
|
|
|
|
|
raise BadRequestError(message=message)
|
2017-12-15 17:13:55 +00:00
|
|
|
|
else:
|
|
|
|
|
|
reply_to = template.get_reply_to_text()
|
2017-11-25 11:31:36 +00:00
|
|
|
|
return reply_to
|
2019-09-05 17:36:02 +01:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def send_pdf_letter_notification(service_id, post_data):
|
|
|
|
|
|
service = dao_fetch_service_by_id(service_id)
|
|
|
|
|
|
|
2020-06-26 14:10:12 +01:00
|
|
|
|
check_service_has_permission(LETTER_TYPE, [
|
|
|
|
|
|
p.permission for p in service.permissions
|
|
|
|
|
|
])
|
2019-09-05 17:36:02 +01:00
|
|
|
|
check_service_over_daily_message_limit(KEY_TYPE_NORMAL, service)
|
|
|
|
|
|
validate_created_by(service, post_data['created_by'])
|
2019-10-22 09:13:01 +01:00
|
|
|
|
validate_and_format_recipient(
|
2020-01-02 16:35:18 +00:00
|
|
|
|
send_to=post_data['recipient_address'],
|
2019-10-22 09:13:01 +01:00
|
|
|
|
key_type=KEY_TYPE_NORMAL,
|
|
|
|
|
|
service=service,
|
|
|
|
|
|
notification_type=LETTER_TYPE,
|
2020-07-28 10:19:46 +01:00
|
|
|
|
allow_guest_list_recipients=False,
|
2019-10-22 09:13:01 +01:00
|
|
|
|
)
|
2019-09-05 17:36:02 +01:00
|
|
|
|
|
2022-04-12 15:51:06 +01:00
|
|
|
|
# notification already exists e.g. if the user clicked send in different tabs
|
|
|
|
|
|
if get_notification_by_id(post_data['file_id']):
|
|
|
|
|
|
return {'id': str(post_data['file_id'])}
|
|
|
|
|
|
|
2019-09-05 17:36:02 +01:00
|
|
|
|
template = get_precompiled_letter_template(service.id)
|
|
|
|
|
|
file_location = 'service-{}/{}.pdf'.format(service.id, post_data['file_id'])
|
|
|
|
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
|
letter = utils_s3download(current_app.config['TRANSIENT_UPLOADED_LETTERS'], file_location)
|
|
|
|
|
|
except S3ObjectNotFound as e:
|
Don't error sending a letter that's sent already
Fixes this error (in Admin):
File "/home/vcap/app/app/notify_client/notification_api_client.py", line 74, in send_precompiled_letter
return self.post(url='/service/{}/send-pdf-letter'.format(service_id), data=data)
File "/home/vcap/app/app/notify_client/__init__.py", line 59, in post
return super().post(*args, **kwargs)
File "/home/vcap/deps/0/python/lib/python3.9/site-packages/notifications_python_client/base.py", line 48, in post
return self.request("POST", url, data=data)
File "/home/vcap/deps/0/python/lib/python3.9/site-packages/notifications_python_client/base.py", line 64, in request
response = self._perform_request(method, url, kwargs)
File "/home/vcap/deps/0/python/lib/python3.9/site-packages/notifications_python_client/base.py", line 118, in _perform_request
raise api_error
notifications_python_client.errors.HTTPError: 500 - Internal server error
Due to this error (in API):
File "/home/vcap/app/app/service/send_notification.py", line 178, in send_pdf_letter_notification
raise e
File "/home/vcap/app/app/service/send_notification.py", line 173, in send_pdf_letter_notification
letter = utils_s3download(current_app.config['TRANSIENT_UPLOADED_LETTERS'], file_location)
File "/home/vcap/deps/0/python/lib/python3.9/site-packages/notifications_utils/s3.py", line 53, in s3download
raise S3ObjectNotFound(error.response, error.operation_name)
notifications_utils.s3.S3ObjectNotFound: An error occurred (NoSuchKey) when calling the GetObject operation: The specified key does not exist.
I checked the DB to verify the letter does actually exist i.e. it
is an instance of the problem we're fixing here.
2022-04-08 17:20:44 +01:00
|
|
|
|
current_app.logger.warning('Letter {}.pdf not in transient {} bucket'.format(
|
2019-09-05 17:36:02 +01:00
|
|
|
|
post_data['file_id'], current_app.config['TRANSIENT_UPLOADED_LETTERS'])
|
|
|
|
|
|
)
|
Don't error sending a letter that's sent already
Fixes this error (in Admin):
File "/home/vcap/app/app/notify_client/notification_api_client.py", line 74, in send_precompiled_letter
return self.post(url='/service/{}/send-pdf-letter'.format(service_id), data=data)
File "/home/vcap/app/app/notify_client/__init__.py", line 59, in post
return super().post(*args, **kwargs)
File "/home/vcap/deps/0/python/lib/python3.9/site-packages/notifications_python_client/base.py", line 48, in post
return self.request("POST", url, data=data)
File "/home/vcap/deps/0/python/lib/python3.9/site-packages/notifications_python_client/base.py", line 64, in request
response = self._perform_request(method, url, kwargs)
File "/home/vcap/deps/0/python/lib/python3.9/site-packages/notifications_python_client/base.py", line 118, in _perform_request
raise api_error
notifications_python_client.errors.HTTPError: 500 - Internal server error
Due to this error (in API):
File "/home/vcap/app/app/service/send_notification.py", line 178, in send_pdf_letter_notification
raise e
File "/home/vcap/app/app/service/send_notification.py", line 173, in send_pdf_letter_notification
letter = utils_s3download(current_app.config['TRANSIENT_UPLOADED_LETTERS'], file_location)
File "/home/vcap/deps/0/python/lib/python3.9/site-packages/notifications_utils/s3.py", line 53, in s3download
raise S3ObjectNotFound(error.response, error.operation_name)
notifications_utils.s3.S3ObjectNotFound: An error occurred (NoSuchKey) when calling the GetObject operation: The specified key does not exist.
I checked the DB to verify the letter does actually exist i.e. it
is an instance of the problem we're fixing here.
2022-04-08 17:20:44 +01:00
|
|
|
|
|
2019-09-05 17:36:02 +01:00
|
|
|
|
raise e
|
|
|
|
|
|
|
|
|
|
|
|
# Getting the page count won't raise an error since admin has already checked the PDF is valid
|
2019-10-16 13:35:30 +01:00
|
|
|
|
page_count = get_page_count(letter.read())
|
|
|
|
|
|
billable_units = get_billable_units_for_letter_page_count(page_count)
|
2019-09-05 17:36:02 +01:00
|
|
|
|
|
|
|
|
|
|
personalisation = {
|
|
|
|
|
|
'address_line_1': post_data['filename']
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
notification = persist_notification(
|
|
|
|
|
|
notification_id=post_data['file_id'],
|
|
|
|
|
|
template_id=template.id,
|
|
|
|
|
|
template_version=template.version,
|
2020-01-02 16:35:18 +00:00
|
|
|
|
recipient=urllib.parse.unquote(post_data['recipient_address']),
|
2019-09-05 17:36:02 +01:00
|
|
|
|
service=service,
|
|
|
|
|
|
personalisation=personalisation,
|
|
|
|
|
|
notification_type=LETTER_TYPE,
|
|
|
|
|
|
api_key_id=None,
|
|
|
|
|
|
key_type=KEY_TYPE_NORMAL,
|
|
|
|
|
|
reference=create_one_off_reference(LETTER_TYPE),
|
|
|
|
|
|
client_reference=post_data['filename'],
|
|
|
|
|
|
created_by_id=post_data['created_by'],
|
|
|
|
|
|
billable_units=billable_units,
|
2020-08-04 15:41:42 +01:00
|
|
|
|
postage=post_data['postage'] or template.postage,
|
2019-09-05 17:36:02 +01:00
|
|
|
|
)
|
|
|
|
|
|
|
Relax lookup of letter PDFs in S3 buckets
Previously we generated the filename we expected a letter PDF to be
stored at in S3, and used that to retrieve it. However, the generated
filename can change over the course of a notification's lifetime e.g.
if the service changes from crown ('.C.') to non-crown ('.N.').
The prefix of the filename is stable: it's based on properties of the
notification - reference and creation - that don't change. This commit
changes the way we interact with letter PDFs in S3:
- Uploading uses the original method to generate the full file name.
The method is renamed to 'generate_' to distinguish it from the new one.
- Downloading uses a new 'find_' method to get the filename using just
its prefix, which makes it agnostic to changes in the filename suffix.
Making this change helps to decouple our code from the requirements DVLA
have on the filenames. While it means more traffic to S3, we rely on S3
in any case to download the files. From experience, we know S3 is highly
reliable and performant, so don't anticipate any issues.
In the tests we favour using moto to mock S3, so that the behaviour is
realistic. There are a couple of places where we just mock the method,
since what it returns isn't important for the test.
Note that, since the new method requires a notification object, we need
to change a query in one place, the columns of which were only selected
to appease the original method to generate a filename.
2021-03-08 15:23:37 +00:00
|
|
|
|
upload_filename = generate_letter_pdf_filename(
|
2019-09-16 14:20:40 +01:00
|
|
|
|
reference=notification.reference,
|
2020-09-21 14:40:22 +01:00
|
|
|
|
created_at=notification.created_at,
|
2020-09-21 13:46:31 +01:00
|
|
|
|
ignore_folder=False,
|
2019-09-05 17:36:02 +01:00
|
|
|
|
postage=notification.postage
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
move_uploaded_pdf_to_letters_bucket(file_location, upload_filename)
|
|
|
|
|
|
|
|
|
|
|
|
return {'id': str(notification.id)}
|