Update the post letter flow - not able to get reduce the dB transactions used in the letter flow. Prioritising the reduction for the SMS/Email flow.

Only update the daily limit cache if the service is in trial mode.
This commit is contained in:
Rebecca Law
2020-06-17 12:11:28 +01:00
parent 21a1b8e8bd
commit a5ed8f2079
5 changed files with 175 additions and 169 deletions

View File

@@ -127,7 +127,6 @@ def persist_notification(
# if simulated create a Notification model to return but do not persist the Notification to the dB
if not simulated:
dao_create_notification(notification)
print("****** COMMIT *******")
# Only keep track of the daily limit for trial mode services.
if service_in_trial_mode and key_type != KEY_TYPE_TEST:
if redis_store.get(redis.daily_limit_cache_key(service_id)):
@@ -168,8 +167,8 @@ def send_notification_to_queue_detached(
"{} {} sent to the {} queue for delivery".format(notification_type,
notification_id,
queue))
def send_notification_to_queue(notification, research_mode, queue=None):
send_notification_to_queue_detached(
notification.key_type, notification.notification_type, notification.id, research_mode, queue

View File

@@ -1,57 +1,10 @@
def create_post_sms_response_from_notification(notification, content, from_number, url_root, scheduled_for):
noti = __create_notification_response(notification, url_root, scheduled_for)
noti['content'] = {
'from_number': from_number,
'body': content
}
return noti
def create_post_email_response_from_notification(notification, content, subject, email_from, url_root, scheduled_for):
noti = __create_notification_response(notification, url_root, scheduled_for)
noti['content'] = {
"from_email": email_from,
"body": content,
"subject": subject
}
return noti
def create_post_letter_response_from_notification(notification, content, subject, url_root, scheduled_for):
noti = __create_notification_response(notification, url_root, scheduled_for)
noti['content'] = {
"body": content,
"subject": subject
}
return noti
def __create_notification_response(notification, url_root, scheduled_for):
return {
"id": notification.id,
"reference": notification.client_reference,
"uri": "{}v2/notifications/{}".format(url_root, str(notification.id)),
'template': {
"id": notification.template_id,
"version": notification.template_version,
"uri": "{}services/{}/templates/{}".format(
url_root,
str(notification.service_id),
str(notification.template_id)
)
},
"scheduled_for": scheduled_for if scheduled_for else None
}
# test detaching notification
def create_post_sms_response_from_notification_detached(
notification_id, client_reference, template_id, template_version, service_id,
def create_post_sms_response_from_notification(
notification_id, client_reference, template_id, template_version, service_id,
content, from_number, url_root, scheduled_for
):
resp = __create_notification_response_detached(
resp = __create_notification_response(
notification_id, client_reference, template_id, template_version, service_id, url_root, scheduled_for
)
resp['content'] = {
@@ -61,10 +14,19 @@ def create_post_sms_response_from_notification_detached(
return resp
def create_post_email_response_from_notification_detached(
notification_id, client_reference, template_id, template_version, service_id, content, subject, email_from,
url_root, scheduled_for):
resp = __create_notification_response_detached(
def create_post_email_response_from_notification(
notification_id,
client_reference,
template_id,
template_version,
service_id,
content,
subject,
email_from,
url_root,
scheduled_for
):
resp = __create_notification_response(
notification_id, client_reference, template_id, template_version, service_id, url_root, scheduled_for
)
resp['content'] = {
@@ -75,11 +37,11 @@ def create_post_email_response_from_notification_detached(
return resp
def create_post_letter_response_from_notification_detached(
def create_post_letter_response_from_notification(
notification_id, client_reference, template_id, template_version, service_id,
content, subject, url_root, scheduled_for
):
resp = __create_notification_response_detached(
resp = __create_notification_response(
notification_id, client_reference, template_id, template_version, service_id, url_root, scheduled_for
)
resp['content'] = {
@@ -89,7 +51,7 @@ def create_post_letter_response_from_notification_detached(
return resp
def __create_notification_response_detached(
def __create_notification_response(
notification_id, client_reference, template_id, template_version, service_id, url_root, scheduled_for
):
return {

View File

@@ -22,7 +22,6 @@ from app.celery.research_mode_tasks import create_fake_letter_response_file
from app.celery.tasks import save_api_email
from app.clients.document_download import DocumentDownloadError
from app.config import QueueNames, TaskNames
from app.dao.notifications_dao import update_notification_status_by_reference
from app.dao.templates_dao import get_precompiled_letter_template
from app.letters.utils import upload_letter_pdf
from app.models import (
@@ -45,7 +44,6 @@ from app.notifications.process_letter_notifications import (
from app.notifications.process_notifications import (
persist_notification,
persist_scheduled_notification,
send_notification_to_queue,
simulated_recipient,
send_notification_to_queue_detached)
from app.notifications.validators import (
@@ -62,11 +60,8 @@ from app.schema_validation import validate
from app.v2.errors import BadRequestError, ValidationError
from app.v2.notifications import v2_notification_blueprint
from app.v2.notifications.create_response import (
create_post_sms_response_from_notification,
create_post_email_response_from_notification,
create_post_letter_response_from_notification,
create_post_sms_response_from_notification_detached, create_post_email_response_from_notification_detached,
create_post_letter_response_from_notification_detached)
create_post_sms_response_from_notification, create_post_email_response_from_notification,
create_post_letter_response_from_notification)
from app.v2.notifications.notification_schemas import (
post_sms_request,
post_email_request,
@@ -109,17 +104,12 @@ def post_precompiled_letter_notification():
letter_data=form,
api_key=api_user,
template=template,
template_with_content=None, # not required for precompiled
reply_to_text=reply_to,
precompiled=True
)
resp = {
'id': notification.id,
'reference': notification.client_reference,
'postage': notification.postage
}
return jsonify(resp), 201
return jsonify(notification), 201
@v2_notification_blueprint.route('/<notification_type>', methods=['POST'])
@@ -152,12 +142,13 @@ def post_notification(notification_type):
)
reply_to = get_reply_to_text(notification_type, form, template)
if notification_type == LETTER_TYPE:
notification = process_letter_notification(
letter_data=form,
api_key=api_user,
template=template,
template_with_content=template_with_content,
reply_to_text=reply_to
)
else:
@@ -174,34 +165,6 @@ def post_notification(notification_type):
return jsonify(notification), 201
def create_response_for_post_notification(notification_id, client_reference, template_id, template_version, service_id,
notification_type, reply_to, scheduled_for,
template_with_content):
if notification_type == SMS_TYPE:
create_resp_partial = functools.partial(
create_post_sms_response_from_notification_detached,
from_number=reply_to,
)
elif notification_type == EMAIL_TYPE:
create_resp_partial = functools.partial(
create_post_email_response_from_notification_detached,
subject=template_with_content.subject,
email_from='{}@{}'.format(authenticated_service.email_from, current_app.config['NOTIFY_EMAIL_DOMAIN']),
)
elif notification_type == LETTER_TYPE:
create_resp_partial = functools.partial(
create_post_letter_response_from_notification_detached,
subject=template_with_content.subject,
)
resp = create_resp_partial(
notification_id, client_reference, template_id, template_version, service_id,
url_root=request.url_root,
scheduled_for=scheduled_for,
content=template_with_content.content_with_placeholders_filled_in,
)
return resp
def process_sms_or_email_notification(
*, form, notification_type, api_key, template, template_process_type, service, reply_to_text=None
):
@@ -221,9 +184,11 @@ def process_sms_or_email_notification(
service,
simulated=simulated
)
if document_download_count:
# We changed personalisation which means we need to update the content
template.values = personalisation
key_type = api_key.key_type
service_in_research_mode = service.resear
service_in_research_mode = service.research_mode
resp = create_response_for_post_notification(
notification_id=notification_id,
client_reference=form.get('reference', None),
@@ -232,7 +197,7 @@ def process_sms_or_email_notification(
service_id=service.id,
notification_type=notification_type,
reply_to=reply_to_text,
scheduled_for=None,
scheduled_for=form.get("scheduled_for", None),
template_with_content=template)
if str(service.id) in current_app.config.get('HIGH_VOLUME_SERVICE') and api_key.key_type == KEY_TYPE_NORMAL \
@@ -242,7 +207,7 @@ def process_sms_or_email_notification(
# the task will then save the notification, then call send_notification_to_queue.
# We know that this team does not use the GET request, but relies on callbacks to get the status updates.
try:
notification = save_email_to_queue(
save_email_to_queue(
form=form,
notification_id=str(notification_id),
notification_type=notification_type,
@@ -253,7 +218,7 @@ def process_sms_or_email_notification(
document_download_count=document_download_count,
reply_to_text=reply_to_text
)
return notification
return resp
except SQSError:
# if SQS cannot put the task on the queue, it's probably because the notification body was too long and it
# went over SQS's 256kb message limit. If so, we
@@ -287,7 +252,7 @@ def process_sms_or_email_notification(
key_type=key_type,
notification_type=notification_type,
notification_id=notification_id,
research_mode=False, # research_mode is a deprecated mode
research_mode=service_in_research_mode, # research_mode is deprecated
queue=queue_name
)
else:
@@ -311,7 +276,7 @@ def save_email_to_queue(
data = {
"id": notification_id,
"template_id": str(template.id),
"template_version": template.version,
"template_version": template._template['version'],
"to": form['email_address'],
"service_id": str(service_id),
"personalisation": personalisation,
@@ -362,7 +327,9 @@ def process_document_uploads(personalisation_data, service, simulated=False):
return personalisation_data, len(file_keys)
def process_letter_notification(*, letter_data, api_key, template, reply_to_text, precompiled=False):
def process_letter_notification(
*, letter_data, api_key, template, template_with_content, reply_to_text, precompiled=False
):
if api_key.key_type == KEY_TYPE_TEAM:
raise BadRequestError(message='Cannot send letters with a team api key', status_code=403)
@@ -375,34 +342,19 @@ def process_letter_notification(*, letter_data, api_key, template, reply_to_text
template=template,
reply_to_text=reply_to_text)
address = PostalAddress.from_personalisation(
letter_data['personalisation'],
allow_international_letters=api_key.service.has_permission(INTERNATIONAL_LETTERS),
)
if not address.has_enough_lines:
raise ValidationError(
message=f'Address must be at least {PostalAddress.MIN_LINES} lines'
)
if address.has_too_many_lines:
raise ValidationError(
message=f'Address must be no more than {PostalAddress.MAX_LINES} lines'
)
if not address.has_valid_last_line:
if address.allow_international_letters:
raise ValidationError(
message=f'Last line of address must be a real UK postcode or another country'
)
raise ValidationError(
message='Must be a real UK postcode'
)
validate_address(api_key, letter_data)
test_key = api_key.key_type == KEY_TYPE_TEST
# if we don't want to actually send the letter, then start it off in SENDING so we don't pick it up
status = NOTIFICATION_CREATED if not test_key else NOTIFICATION_SENDING
status = NOTIFICATION_CREATED
if test_key:
# if we don't want to actually send the letter, then start it off in SENDING so we don't pick it up
if current_app.config['NOTIFY_ENVIRONMENT'] in ['preview', 'development']:
status = NOTIFICATION_SENDING
# mark test letter as delivered and do not create a fake response later
else:
status = NOTIFICATION_DELIVERED
queue = QueueNames.CREATE_LETTERS_PDF if not test_key else QueueNames.RESEARCH_MODE
notification = create_letter_notification(letter_data=letter_data,
@@ -416,16 +368,46 @@ def process_letter_notification(*, letter_data, api_key, template, reply_to_text
queue=queue
)
if test_key:
if current_app.config['NOTIFY_ENVIRONMENT'] in ['preview', 'development']:
create_fake_letter_response_file.apply_async(
(notification.reference,),
queue=queue
)
else:
update_notification_status_by_reference(notification.reference, NOTIFICATION_DELIVERED)
if test_key and current_app.config['NOTIFY_ENVIRONMENT'] in ['preview', 'development']:
create_fake_letter_response_file.apply_async(
(notification.reference,),
queue=queue
)
resp = create_response_for_post_notification(
notification_id=notification.id,
client_reference=notification.client_reference,
template_id=notification.template_id,
template_version=notification.template_version,
notification_type=notification.notification_type,
reply_to=reply_to_text,
scheduled_for=letter_data.get('scheduled_for', None),
service_id=notification.service_id,
template_with_content=template_with_content
)
return resp
return notification
def validate_address(api_key, letter_data):
address = PostalAddress.from_personalisation(
letter_data['personalisation'],
allow_international_letters=api_key.service.has_permission(INTERNATIONAL_LETTERS),
)
if not address.has_enough_lines:
raise ValidationError(
message=f'Address must be at least {PostalAddress.MIN_LINES} lines'
)
if address.has_too_many_lines:
raise ValidationError(
message=f'Address must be no more than {PostalAddress.MAX_LINES} lines'
)
if not address.has_valid_last_line:
if address.allow_international_letters:
raise ValidationError(
message=f'Last line of address must be a real UK postcode or another country'
)
raise ValidationError(
message='Must be a real UK postcode'
)
def process_precompiled_letter_notifications(*, letter_data, api_key, template, reply_to_text):
@@ -441,6 +423,12 @@ def process_precompiled_letter_notifications(*, letter_data, api_key, template,
status=status,
reply_to_text=reply_to_text)
resp = {
'id': notification.id,
'reference': notification.client_reference,
'postage': notification.postage
}
filename = upload_letter_pdf(notification, letter_content, precompiled=True)
current_app.logger.info('Calling task scan-file for {}'.format(filename))
@@ -459,7 +447,7 @@ def process_precompiled_letter_notifications(*, letter_data, api_key, template,
queue=QueueNames.LETTERS
)
return notification
return resp
def get_reply_to_text(notification_type, form, template):
@@ -484,3 +472,31 @@ def get_reply_to_text(notification_type, form, template):
reply_to = template.get_reply_to_text()
return reply_to
def create_response_for_post_notification(notification_id, client_reference, template_id, template_version, service_id,
notification_type, reply_to, scheduled_for,
template_with_content):
if notification_type == SMS_TYPE:
create_resp_partial = functools.partial(
create_post_sms_response_from_notification,
from_number=reply_to,
)
elif notification_type == EMAIL_TYPE:
create_resp_partial = functools.partial(
create_post_email_response_from_notification,
subject=template_with_content.subject,
email_from='{}@{}'.format(authenticated_service.email_from, current_app.config['NOTIFY_EMAIL_DOMAIN']),
)
elif notification_type == LETTER_TYPE:
create_resp_partial = functools.partial(
create_post_letter_response_from_notification,
subject=template_with_content.subject,
)
resp = create_resp_partial(
notification_id, client_reference, template_id, template_version, service_id,
url_root=request.url_root,
scheduled_for=scheduled_for,
content=template_with_content.content_with_placeholders_filled_in,
)
return resp

View File

@@ -23,7 +23,7 @@ from app.notifications.process_notifications import (
)
from notifications_utils.recipients import validate_and_format_phone_number, validate_and_format_email_address
from app.v2.errors import BadRequestError
from tests.app.db import create_service, create_template
from tests.app.db import create_service, create_template, create_api_key
def test_create_content_for_notification_passes(sample_email_template):
@@ -51,8 +51,7 @@ def test_create_content_for_notification_allows_additional_personalisation(sampl
@freeze_time("2016-01-01 11:09:00.061258")
def test_persist_notification_creates_and_save_to_db(sample_template, sample_api_key, sample_job, mocker):
mocked_redis = mocker.patch('app.notifications.process_notifications.redis_store.get')
def test_persist_notification_creates_and_save_to_db(sample_template, sample_api_key, sample_job):
assert Notification.query.count() == 0
assert NotificationHistory.query.count() == 0
@@ -91,8 +90,6 @@ def test_persist_notification_creates_and_save_to_db(sample_template, sample_api
assert notification_from_db.created_by_id == notification.created_by_id
assert notification_from_db.reply_to_text == sample_template.service.get_default_sms_sender()
mocked_redis.assert_called_once_with(str(sample_template.service_id) + "-2016-01-01-count")
def test_persist_notification_throws_exception_when_missing_template(sample_api_key):
assert Notification.query.count() == 0
@@ -157,10 +154,9 @@ def test_persist_notification_does_not_increment_cache_if_test_key(
@freeze_time("2016-01-01 11:09:00.061258")
def test_persist_notification_with_optionals(sample_job, sample_api_key, mocker):
def test_persist_notification_with_optionals(sample_job, sample_api_key):
assert Notification.query.count() == 0
assert NotificationHistory.query.count() == 0
mocked_redis = mocker.patch('app.notifications.process_notifications.redis_store.get')
n_id = uuid.uuid4()
created_at = datetime.datetime(2016, 11, 11, 16, 8, 18)
persist_notification(
@@ -186,7 +182,7 @@ def test_persist_notification_with_optionals(sample_job, sample_api_key, mocker)
persisted_notification.job_id == sample_job.id
assert persisted_notification.job_row_number == 10
assert persisted_notification.created_at == created_at
mocked_redis.assert_called_once_with(str(sample_job.service_id) + "-2016-01-01-count")
assert persisted_notification.client_reference == "ref from client"
assert persisted_notification.reference is None
assert persisted_notification.international is False
@@ -197,44 +193,77 @@ def test_persist_notification_with_optionals(sample_job, sample_api_key, mocker)
@freeze_time("2016-01-01 11:09:00.061258")
def test_persist_notification_doesnt_touch_cache_for_old_keys_that_dont_exist(sample_template, sample_api_key, mocker):
def test_persist_notification_doesnt_touch_cache_for_old_keys_that_dont_exist(notify_db_session, mocker):
service = create_service(restricted=True)
template = create_template(service=service)
api_key = create_api_key(service=service)
mock_incr = mocker.patch('app.notifications.process_notifications.redis_store.incr')
mocker.patch('app.notifications.process_notifications.redis_store.get', return_value=None)
mocker.patch('app.notifications.process_notifications.redis_store.get_all_from_hash', return_value=None)
persist_notification(
template_id=sample_template.id,
template_version=sample_template.version,
template_id=template.id,
template_version=template.version,
recipient='+447111111111',
service=sample_template.service,
service=template.service,
personalisation={},
notification_type='sms',
api_key_id=sample_api_key.id,
key_type=sample_api_key.key_type,
api_key_id=api_key.id,
key_type=api_key.key_type,
reference="ref"
)
mock_incr.assert_not_called()
@freeze_time("2016-01-01 11:09:00.061258")
def test_persist_notification_increments_cache_if_key_exists(sample_template, sample_api_key, mocker):
def test_persist_notification_increments_cache_if_key_exists_and_for_trial_service(
notify_db_session, mocker
):
service = create_service(restricted=True)
template = create_template(service=service)
api_key = create_api_key(service=service)
mock_incr = mocker.patch('app.notifications.process_notifications.redis_store.incr')
mocker.patch('app.notifications.process_notifications.redis_store.get', return_value=1)
mocker.patch('app.notifications.process_notifications.redis_store.get_all_from_hash',
return_value={sample_template.id, 1})
return_value={template.id, 1})
persist_notification(
template_id=sample_template.id,
template_version=sample_template.version,
template_id=template.id,
template_version=template.version,
recipient='+447111111122',
service=sample_template.service,
service=template.service,
personalisation={},
notification_type='sms',
api_key_id=sample_api_key.id,
key_type=sample_api_key.key_type,
api_key_id=api_key.id,
key_type=api_key.key_type,
reference="ref2")
mock_incr.assert_called_once_with(str(sample_template.service_id) + "-2016-01-01-count", )
mock_incr.assert_called_once_with(str(service.id) + "-2016-01-01-count", )
def test_persist_notification_does_not_increments_cache_live_service(
notify_db_session, mocker
):
service = create_service(restricted=False)
template = create_template(service=service)
api_key = create_api_key(service=service)
mock_incr = mocker.patch('app.notifications.process_notifications.redis_store.incr')
mocker.patch('app.notifications.process_notifications.redis_store.get', return_value=1)
mocker.patch('app.notifications.process_notifications.redis_store.get_all_from_hash',
return_value={template.id, 1})
persist_notification(
template_id=template.id,
template_version=template.version,
recipient='+447111111122',
service=template.service,
personalisation={},
notification_type='sms',
api_key_id=api_key.id,
key_type=api_key.key_type,
reference="ref2")
assert not mock_incr.called
@pytest.mark.parametrize((

View File

@@ -426,7 +426,7 @@ def test_returns_a_429_limit_exceeded_if_rate_limit_exceeded(
):
sample = create_template(service=sample_service, template_type=notification_type)
persist_mock = mocker.patch('app.v2.notifications.post_notifications.persist_notification')
deliver_mock = mocker.patch('app.v2.notifications.post_notifications.send_notification_to_queue')
deliver_mock = mocker.patch('app.v2.notifications.post_notifications.send_notification_to_queue_detached')
mocker.patch(
'app.v2.notifications.post_notifications.check_rate_limiting',
side_effect=RateLimitError("LIMIT", "INTERVAL", "TYPE"))