2017-07-20 17:56:51 +01:00
|
|
|
import uuid
|
2016-11-11 14:56:33 +00:00
|
|
|
from datetime import datetime
|
|
|
|
|
|
2016-10-27 17:34:54 +01:00
|
|
|
from flask import current_app
|
2016-10-27 11:46:37 +01:00
|
|
|
|
2017-09-26 09:56:09 +01:00
|
|
|
from notifications_utils.clients import redis
|
2017-04-27 12:41:10 +01:00
|
|
|
from notifications_utils.recipients import (
|
|
|
|
|
get_international_phone_info,
|
2017-05-23 15:45:11 +01:00
|
|
|
validate_and_format_phone_number,
|
|
|
|
|
format_email_address
|
2017-04-27 12:41:10 +01:00
|
|
|
)
|
2019-01-15 14:38:45 +00:00
|
|
|
from notifications_utils.timezones import convert_bst_to_utc
|
2017-04-27 12:41:10 +01:00
|
|
|
|
2017-02-14 14:22:52 +00:00
|
|
|
from app import redis_store
|
2016-10-27 17:34:54 +01:00
|
|
|
from app.celery import provider_tasks
|
2020-05-06 18:23:56 +01:00
|
|
|
from app.celery.letters_pdf_tasks import get_pdf_for_templated_letter
|
2017-07-19 13:50:29 +01:00
|
|
|
from app.config import QueueNames
|
2017-12-01 15:30:18 +00:00
|
|
|
|
2017-11-29 16:47:23 +00:00
|
|
|
from app.models import (
|
|
|
|
|
EMAIL_TYPE,
|
|
|
|
|
KEY_TYPE_TEST,
|
|
|
|
|
SMS_TYPE,
|
2018-09-19 16:39:39 +01:00
|
|
|
LETTER_TYPE,
|
2017-11-29 16:47:23 +00:00
|
|
|
NOTIFICATION_CREATED,
|
|
|
|
|
Notification,
|
2019-02-05 12:48:40 +00:00
|
|
|
ScheduledNotification
|
2017-11-29 16:47:23 +00:00
|
|
|
)
|
|
|
|
|
from app.dao.notifications_dao import (
|
|
|
|
|
dao_create_notification,
|
2019-05-30 10:37:57 +01:00
|
|
|
dao_delete_notifications_by_id,
|
2017-11-29 16:47:23 +00:00
|
|
|
dao_created_scheduled_notification
|
|
|
|
|
)
|
2017-12-01 15:30:18 +00:00
|
|
|
|
2017-06-19 14:58:38 +01:00
|
|
|
from app.v2.errors import BadRequestError
|
2016-10-27 11:46:37 +01:00
|
|
|
|
|
|
|
|
|
|
|
|
|
def create_content_for_notification(template, personalisation):
|
2020-04-06 14:25:43 +01:00
|
|
|
template_object = template._as_utils_template_with_personalisation(personalisation)
|
2016-10-28 17:10:00 +01:00
|
|
|
check_placeholders(template_object)
|
2016-10-27 11:46:37 +01:00
|
|
|
|
|
|
|
|
return template_object
|
|
|
|
|
|
|
|
|
|
|
2016-10-28 17:10:00 +01:00
|
|
|
def check_placeholders(template_object):
|
|
|
|
|
if template_object.missing_data:
|
2017-10-23 14:41:49 +01:00
|
|
|
message = 'Missing personalisation: {}'.format(", ".join(template_object.missing_data))
|
2016-10-31 12:22:26 +00:00
|
|
|
raise BadRequestError(fields=[{'template': message}], message=message)
|
2016-10-28 17:10:00 +01:00
|
|
|
|
|
|
|
|
|
2017-04-27 12:41:10 +01:00
|
|
|
def persist_notification(
|
2017-07-07 17:10:25 +01:00
|
|
|
*,
|
2017-04-27 12:41:10 +01:00
|
|
|
template_id,
|
|
|
|
|
template_version,
|
|
|
|
|
recipient,
|
|
|
|
|
service,
|
|
|
|
|
personalisation,
|
|
|
|
|
notification_type,
|
|
|
|
|
api_key_id,
|
|
|
|
|
key_type,
|
2017-05-17 13:27:05 +01:00
|
|
|
created_at=None,
|
2017-04-27 12:41:10 +01:00
|
|
|
job_id=None,
|
|
|
|
|
job_row_number=None,
|
|
|
|
|
reference=None,
|
|
|
|
|
client_reference=None,
|
|
|
|
|
notification_id=None,
|
2017-06-16 16:30:03 +01:00
|
|
|
simulated=False,
|
2017-09-26 09:56:09 +01:00
|
|
|
created_by_id=None,
|
2017-11-23 14:55:49 +00:00
|
|
|
status=NOTIFICATION_CREATED,
|
2018-09-20 14:47:24 +01:00
|
|
|
reply_to_text=None,
|
2019-01-09 17:49:19 +00:00
|
|
|
billable_units=None,
|
2019-01-14 17:45:56 +00:00
|
|
|
postage=None,
|
2020-02-12 16:07:07 +00:00
|
|
|
template_postage=None,
|
2020-02-27 13:52:02 +00:00
|
|
|
document_download_count=None
|
2017-04-27 12:41:10 +01:00
|
|
|
):
|
2017-05-17 13:25:40 +01:00
|
|
|
notification_created_at = created_at or datetime.utcnow()
|
2017-07-26 15:57:30 +01:00
|
|
|
if not notification_id:
|
2017-07-20 17:56:51 +01:00
|
|
|
notification_id = uuid.uuid4()
|
2016-11-14 14:41:32 +00:00
|
|
|
notification = Notification(
|
2016-11-25 17:32:01 +00:00
|
|
|
id=notification_id,
|
2016-11-11 14:56:33 +00:00
|
|
|
template_id=template_id,
|
|
|
|
|
template_version=template_version,
|
2016-11-14 14:41:32 +00:00
|
|
|
to=recipient,
|
2016-12-19 16:45:18 +00:00
|
|
|
service_id=service.id,
|
|
|
|
|
service=service,
|
2016-11-11 14:56:33 +00:00
|
|
|
personalisation=personalisation,
|
|
|
|
|
notification_type=notification_type,
|
|
|
|
|
api_key_id=api_key_id,
|
|
|
|
|
key_type=key_type,
|
2017-05-17 13:25:40 +01:00
|
|
|
created_at=notification_created_at,
|
2016-11-11 14:56:33 +00:00
|
|
|
job_id=job_id,
|
2016-11-16 15:44:16 +00:00
|
|
|
job_row_number=job_row_number,
|
2017-04-12 17:56:55 +01:00
|
|
|
client_reference=client_reference,
|
2017-06-23 15:56:47 +01:00
|
|
|
reference=reference,
|
2017-09-26 09:56:09 +01:00
|
|
|
created_by_id=created_by_id,
|
2017-11-23 14:55:49 +00:00
|
|
|
status=status,
|
|
|
|
|
reply_to_text=reply_to_text,
|
2020-02-12 16:07:07 +00:00
|
|
|
billable_units=billable_units,
|
|
|
|
|
document_download_count=document_download_count,
|
2016-11-11 14:56:33 +00:00
|
|
|
)
|
2017-04-27 12:41:10 +01:00
|
|
|
|
|
|
|
|
if notification_type == SMS_TYPE:
|
|
|
|
|
formatted_recipient = validate_and_format_phone_number(recipient, international=True)
|
|
|
|
|
recipient_info = get_international_phone_info(formatted_recipient)
|
2017-05-23 14:47:55 +01:00
|
|
|
notification.normalised_to = formatted_recipient
|
2017-04-27 12:41:10 +01:00
|
|
|
notification.international = recipient_info.international
|
|
|
|
|
notification.phone_prefix = recipient_info.country_prefix
|
|
|
|
|
notification.rate_multiplier = recipient_info.billable_units
|
2017-05-23 15:45:11 +01:00
|
|
|
elif notification_type == EMAIL_TYPE:
|
|
|
|
|
notification.normalised_to = format_email_address(notification.to)
|
2018-09-19 16:39:39 +01:00
|
|
|
elif notification_type == LETTER_TYPE:
|
2019-02-05 12:48:40 +00:00
|
|
|
notification.postage = postage or template_postage
|
2020-04-22 10:06:25 +01:00
|
|
|
notification.normalised_to = ''.join(notification.to.split()).lower()
|
2017-04-27 12:41:10 +01:00
|
|
|
|
|
|
|
|
# if simulated create a Notification model to return but do not persist the Notification to the dB
|
2017-01-17 12:08:24 +00:00
|
|
|
if not simulated:
|
2017-03-30 10:46:23 +01:00
|
|
|
dao_create_notification(notification)
|
2017-03-30 13:43:44 +01:00
|
|
|
if key_type != KEY_TYPE_TEST:
|
|
|
|
|
if redis_store.get(redis.daily_limit_cache_key(service.id)):
|
|
|
|
|
redis_store.incr(redis.daily_limit_cache_key(service.id))
|
add new redis template usage per day key
We've run into issues with redis expiring keys while we try and write
to them - short lived redis TTLs aren't really sustainable for keys
where we mutate the state. Template usage is a hash contained in redis
where we increment a count keyed by template_id each time a message is
sent for that template. But if the key expires, hincrby (redis command
for incrementing a value in a hash) will re-create an empty hash.
This is no good, as we need the hash to be populated with the last
seven days worth of data, which we then increment further. We can't
tell whether the hincrby created the key, so a different approach
entirely was needed:
* New redis key: <service_id>-template-usage-<YYYY-MM-DD>. Note: This
YYYY-MM-DD is BTC time so it lines up nicely with ft_billing table
* Incremented to from process_notification - if it doesn't exist yet,
it'll be created then.
* Expiry set to 8 days every time it's incremented to.
Then, at read time, we'll just read the last eight days of keys from
Redis, and sum them up. This works because we're only ever incrementing
from that one place - never setting wholesale, never recreating the
data from scratch. So we know that if the data is in redis, then it is
good and accurate data.
One thing we *don't* know and *cannot* reason about is what no key in
redis means. It could be either of:
* This is the first message that the service has sent today.
* The key was deleted from redis for some reason.
Since we set the TTL to so long, we'll never be writing to a key that
previously expired. But if there is a redis (or operator) error and the
key is deleted, then we'll have bad data - after any data loss we'll
have to rebuild the data.
2018-03-29 13:55:22 +01:00
|
|
|
|
2017-01-18 09:56:26 +00:00
|
|
|
current_app.logger.info(
|
2017-05-17 13:25:40 +01:00
|
|
|
"{} {} created at {}".format(notification_type, notification_id, notification_created_at)
|
2017-01-18 09:56:26 +00:00
|
|
|
)
|
2016-10-27 17:34:54 +01:00
|
|
|
return notification
|
2016-10-25 18:04:03 +01:00
|
|
|
|
|
|
|
|
|
2016-12-09 12:10:42 +00:00
|
|
|
def send_notification_to_queue(notification, research_mode, queue=None):
|
|
|
|
|
if research_mode or notification.key_type == KEY_TYPE_TEST:
|
2017-05-25 10:51:49 +01:00
|
|
|
queue = QueueNames.RESEARCH_MODE
|
2016-12-09 12:10:42 +00:00
|
|
|
|
|
|
|
|
if notification.notification_type == SMS_TYPE:
|
2017-07-20 16:17:04 +01:00
|
|
|
if not queue:
|
|
|
|
|
queue = QueueNames.SEND_SMS
|
2016-12-09 12:10:42 +00:00
|
|
|
deliver_task = provider_tasks.deliver_sms
|
|
|
|
|
if notification.notification_type == EMAIL_TYPE:
|
2017-07-20 16:17:04 +01:00
|
|
|
if not queue:
|
|
|
|
|
queue = QueueNames.SEND_EMAIL
|
2016-12-09 12:10:42 +00:00
|
|
|
deliver_task = provider_tasks.deliver_email
|
2018-10-31 14:30:46 +00:00
|
|
|
if notification.notification_type == LETTER_TYPE:
|
|
|
|
|
if not queue:
|
|
|
|
|
queue = QueueNames.CREATE_LETTERS_PDF
|
2020-05-06 18:23:56 +01:00
|
|
|
deliver_task = get_pdf_for_templated_letter
|
2016-12-09 12:10:42 +00:00
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
deliver_task.apply_async([str(notification.id)], queue=queue)
|
2017-06-19 14:58:38 +01:00
|
|
|
except Exception:
|
2019-05-30 10:37:57 +01:00
|
|
|
dao_delete_notifications_by_id(notification.id)
|
2017-06-19 14:58:38 +01:00
|
|
|
raise
|
2016-10-27 17:34:54 +01:00
|
|
|
|
As Notify matures we probably need less logging, especially to report happy path events.
This PR is a proposal to reduce the average messages we see for a single notification from about 7 messages to 2.
Messaging would change to something like this:
February 2nd 2018, 15:39:05.885 Full delivery response from Firetext for notification: 8eda51d5-cd82-4569-bfc9-d5570cdf2126
{'status': ['0'], 'reference': ['8eda51d5-cd82-4569-bfc9-d5570cdf2126'], 'time': ['2018-02-02 15:39:01'], 'code': ['000']}
February 2nd 2018, 15:39:05.885 Firetext callback return status of 0 for reference: 8eda51d5-cd82-4569-bfc9-d5570cdf2126
February 2nd 2018, 15:38:57.727 SMS 8eda51d5-cd82-4569-bfc9-d5570cdf2126 sent to provider firetext at 2018-02-02 15:38:56.716814
February 2nd 2018, 15:38:56.727 Starting sending SMS 8eda51d5-cd82-4569-bfc9-d5570cdf2126 to provider at 2018-02-02 15:38:56.408181
February 2nd 2018, 15:38:56.727 Firetext request for 8eda51d5-cd82-4569-bfc9-d5570cdf2126 finished in 0.30376038211397827
February 2nd 2018, 15:38:49.449 sms 8eda51d5-cd82-4569-bfc9-d5570cdf2126 created at 2018-02-02 15:38:48.439113
February 2nd 2018, 15:38:49.449 sms 8eda51d5-cd82-4569-bfc9-d5570cdf2126 sent to the priority-tasks queue for delivery
To somthing like this:
February 2nd 2018, 15:39:05.885 Firetext callback return status of 0 for reference: 8eda51d5-cd82-4569-bfc9-d5570cdf2126
February 2nd 2018, 15:38:49.449 sms 8eda51d5-cd82-4569-bfc9-d5570cdf2126 created at 2018-02-02 15:38:48.439113
2018-02-02 15:55:25 +00:00
|
|
|
current_app.logger.debug(
|
2017-01-18 09:56:26 +00:00
|
|
|
"{} {} sent to the {} queue for delivery".format(notification.notification_type,
|
|
|
|
|
notification.id,
|
|
|
|
|
queue))
|
2017-01-17 12:08:24 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
def simulated_recipient(to_address, notification_type):
|
2017-04-26 15:56:45 +01:00
|
|
|
if notification_type == SMS_TYPE:
|
|
|
|
|
formatted_simulated_numbers = [
|
|
|
|
|
validate_and_format_phone_number(number) for number in current_app.config['SIMULATED_SMS_NUMBERS']
|
|
|
|
|
]
|
|
|
|
|
return to_address in formatted_simulated_numbers
|
|
|
|
|
else:
|
|
|
|
|
return to_address in current_app.config['SIMULATED_EMAIL_ADDRESSES']
|
2017-05-15 17:27:38 +01:00
|
|
|
|
|
|
|
|
|
|
|
|
|
def persist_scheduled_notification(notification_id, scheduled_for):
|
2017-05-22 14:15:35 +01:00
|
|
|
scheduled_datetime = convert_bst_to_utc(datetime.strptime(scheduled_for, "%Y-%m-%d %H:%M"))
|
2017-05-15 17:27:38 +01:00
|
|
|
scheduled_notification = ScheduledNotification(notification_id=notification_id,
|
2017-05-17 15:06:15 +01:00
|
|
|
scheduled_for=scheduled_datetime)
|
2017-05-15 17:27:38 +01:00
|
|
|
dao_created_scheduled_notification(scheduled_notification)
|