darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-09-11 18:38:14 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into pyup-update-pyjwt-1.5.3-to-1.6.0

Browse Source
This commit is contained in:
kentsanggds
2018-03-09 14:21:10 +00:00
committed by GitHub
parent 71bf8d4a22 d9a0c8e7f0
commit d393342430
47 changed files with 1337 additions and 458 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+27
View File
@@ -115,3 +115,30 @@ cf run-task notify-api "flask command purge_functional_test_data -u <functional
```
All commands and command options have a --help command if you need more information.
## To create a new worker app
You need to:
1. Create a new entry for your app in manifest-delivery-base.yml ([example](https://github.com/alphagov/notifications-api/commit/131495125e5dfb181010c8595b11b34ab412fc37#diff-a1885d77ffd0a5cb168590428871cd9e))
1. Update the jenkins deployment job in the notifications-aws repo ([example](https://github.com/alphagov/notifications-aws/commit/69cf9912bd638bce088d4845e4b0a3b11a2cb74c#diff-17e034fe6186f2717b77ba277e0a5828))
1. Add the new worker's log group to the list of logs groups we get alerts about and we ship them to kibana ([example](https://github.com/alphagov/notifications-aws/commit/69cf9912bd638bce088d4845e4b0a3b11a2cb74c#diff-501ffa3502adce988e810875af546b97))
1. Optionally add it to the autoscaler ([example](https://github.com/alphagov/notifications-paas-autoscaler/commit/16d4cd0bdc851da2fab9fad1c9130eb94acf3d15))
**Important:**
Before pushing the deployment change on jenkins, read below about the first time deployment.
### First time deployment of your new worker
Our deployment flow requires that the app is present in order to proceed with the deployment.
This means that the first deployment of your app must happen manually.
To do this:
1. Ensure your code is backwards compatible
1. From the root of this repo run `CF_APP=<APP_NAME> make <cf-space> cf-push`
Once this is done, you can push your deployment changes to jenkins to have your app deployed on every deployment.
app/celery/scheduled_tasks.py
+13 -5
View File
@@ -213,19 +213,21 @@ def timeout_notifications():
@statsd(namespace="tasks")
def send_daily_performance_platform_stats():
if performance_platform_client.active:
send_total_sent_notifications_to_performance_platform()
yesterday = datetime.utcnow() - timedelta(days=1)
send_total_sent_notifications_to_performance_platform(yesterday)
processing_time.send_processing_time_to_performance_platform()
def send_total_sent_notifications_to_performance_platform():
count_dict = total_sent_notifications.get_total_sent_notifications_yesterday()
def send_total_sent_notifications_to_performance_platform(day):
count_dict = total_sent_notifications.get_total_sent_notifications_for_day(day)
email_sent_count = count_dict.get('email').get('count')
sms_sent_count = count_dict.get('sms').get('count')
letter_sent_count = count_dict.get('letter').get('count')
start_date = count_dict.get('start_date')
current_app.logger.info(
"Attempting to update performance platform for date {} with email count {} and sms count {}"
.format(start_date, email_sent_count, sms_sent_count)
"Attempting to update Performance Platform for {} with {} emails, {} text messages and {} letters"
.format(start_date, email_sent_count, sms_sent_count, letter_sent_count)
)
total_sent_notifications.send_total_notifications_sent_for_day_stats(
@@ -240,6 +242,12 @@ def send_total_sent_notifications_to_performance_platform():
email_sent_count
)
total_sent_notifications.send_total_notifications_sent_for_day_stats(
start_date,
'letter',
letter_sent_count
)
@notify_celery.task(name='switch-current-sms-provider-on-slow-delivery')
@statsd(namespace="tasks")
app/celery/service_callback_tasks.py
+89 -23
View File
@@ -6,6 +6,7 @@ from app import (
db,
DATETIME_FORMAT,
notify_celery,
encryption
)
from app.dao.notifications_dao import (
get_notification_by_id,
@@ -23,29 +24,83 @@ from app.config import QueueNames
@notify_celery.task(bind=True, name="send-delivery-status", max_retries=5, default_retry_delay=300)
@statsd(namespace="tasks")
def send_delivery_status_to_service(self, notification_id):
# TODO: do we need to do rate limit this?
notification = get_notification_by_id(notification_id)
service_callback_api = get_service_callback_api_for_service(service_id=notification.service_id)
if not service_callback_api:
# No delivery receipt API info set
return
def send_delivery_status_to_service(self, notification_id,
encrypted_status_update=None
):
if not encrypted_status_update:
process_update_with_notification_id(self, notification_id=notification_id)
else:
try:
status_update = encryption.decrypt(encrypted_status_update)
# Release DB connection before performing an external HTTP request
db.session.close()
data = {
"id": str(notification_id),
"reference": status_update['notification_client_reference'],
"to": status_update['notification_to'],
"status": status_update['notification_status'],
"created_at": status_update['notification_created_at'],
"completed_at": status_update['notification_updated_at'],
"sent_at": status_update['notification_sent_at'],
"notification_type": status_update['notification_type']
}
data = {
"id": str(notification_id),
"reference": str(notification.client_reference),
"to": notification.to,
"status": notification.status,
"created_at": notification.created_at.strftime(DATETIME_FORMAT), # the time service sent the request
"completed_at": notification.updated_at.strftime(DATETIME_FORMAT), # the last time the status was updated
"sent_at": notification.sent_at.strftime(DATETIME_FORMAT), # the time the email was sent
"notification_type": notification.notification_type
}
response = request(
method="POST",
url=status_update['service_callback_api_url'],
data=json.dumps(data),
headers={
'Content-Type': 'application/json',
'Authorization': 'Bearer {}'.format(status_update['service_callback_api_bearer_token'])
},
timeout=60
)
current_app.logger.info('send_delivery_status_to_service sending {} to {}, response {}'.format(
notification_id,
status_update['service_callback_api_url'],
response.status_code
))
response.raise_for_status()
except RequestException as e:
current_app.logger.warning(
"send_delivery_status_to_service request failed for service_id: {} and url: {}. exc: {}".format(
notification_id,
status_update['service_callback_api_url'],
e
)
)
if not isinstance(e, HTTPError) or e.response.status_code >= 500:
try:
self.retry(queue=QueueNames.RETRY)
except self.MaxRetriesExceededError:
current_app.logger.exception(
"""Retry: send_delivery_status_to_service has retried the max num of times
for notification: {}""".format(notification_id)
)
def process_update_with_notification_id(self, notification_id):
retry = False
try:
notification = get_notification_by_id(notification_id)
service_callback_api = get_service_callback_api_for_service(service_id=notification.service_id)
if not service_callback_api:
# No delivery receipt API info set
return
# Release DB connection before performing an external HTTP request
db.session.close()
data = {
"id": str(notification_id),
"reference": str(notification.client_reference),
"to": notification.to,
"status": notification.status,
"created_at": notification.created_at.strftime(DATETIME_FORMAT),
"completed_at": notification.updated_at.strftime(DATETIME_FORMAT),
"sent_at": notification.sent_at.strftime(DATETIME_FORMAT),
"notification_type": notification.notification_type
}
response = request(
method="POST",
url=service_callback_api.url,
@@ -71,7 +126,18 @@ def send_delivery_status_to_service(self, notification_id):
)
)
if not isinstance(e, HTTPError) or e.response.status_code >= 500:
try:
self.retry(queue=QueueNames.RETRY)
except self.MaxRetriesExceededError:
current_app.logger.exception('Retry: send_delivery_status_to_service has retried the max num of times')
retry = True
except Exception as e:
current_app.logger.exception(
'Unhandled exception when sending callback for notification {}'.format(notification_id)
)
retry = True
if retry:
try:
self.retry(queue=QueueNames.RETRY)
except self.MaxRetriesExceededError:
current_app.logger.exception(
"""Retry: send_delivery_status_to_service has retried the max num of times
for notification: {}""".format(notification_id)
)
app/celery/statistics_tasks.py
-11
View File
@@ -10,20 +10,9 @@ from app.dao.statistics_dao import (
update_job_stats_outcome_count
)
from app.dao.notifications_dao import get_notification_by_id
from app.models import NOTIFICATION_STATUS_TYPES_COMPLETED
from app.config import QueueNames
def create_initial_notification_statistic_tasks(notification):
if notification.job_id and notification.status:
record_initial_job_statistics.apply_async((str(notification.id),), queue=QueueNames.STATISTICS)
def create_outcome_notification_statistic_tasks(notification):
if notification.job_id and notification.status in NOTIFICATION_STATUS_TYPES_COMPLETED:
record_outcome_job_statistics.apply_async((str(notification.id),), queue=QueueNames.STATISTICS)
@worker_process_shutdown.connect
def worker_process_shutdown(sender, signal, pid, exitcode):
current_app.logger.info('Statistics worker shutdown: PID: {} Exitcode: {}'.format(pid, exitcode))
app/celery/tasks.py
+32 -1
View File
@@ -1,6 +1,6 @@
import json
from datetime import datetime
from collections import namedtuple
from collections import namedtuple, defaultdict
from celery.signals import worker_process_shutdown
from flask import current_app
@@ -31,6 +31,7 @@ from app import (
from app.aws import s3
from app.celery import provider_tasks, letters_pdf_tasks, research_mode_tasks
from app.config import QueueNames
from app.dao.daily_sorted_letter_dao import dao_create_or_update_daily_sorted_letter
from app.dao.inbound_sms_dao import dao_get_inbound_sms_by_id
from app.dao.jobs_dao import (
dao_update_job,
@@ -66,9 +67,11 @@ from app.models import (
NOTIFICATION_TEMPORARY_FAILURE,
NOTIFICATION_TECHNICAL_FAILURE,
SMS_TYPE,
DailySortedLetter,
)
from app.notifications.process_notifications import persist_notification
from app.service.utils import service_allowed_to_send_to
from app.utils import convert_utc_to_bst
@worker_process_shutdown.connect
@@ -404,6 +407,7 @@ def get_template_class(template_type):
def update_letter_notifications_statuses(self, filename):
bucket_location = '{}-ftp'.format(current_app.config['NOTIFY_EMAIL_DOMAIN'])
response_file_content = s3.get_s3_file(bucket_location, filename)
sorted_letter_counts = defaultdict(int)
try:
notification_updates = process_updates_from_file(response_file_content)
@@ -414,6 +418,7 @@ def update_letter_notifications_statuses(self, filename):
for update in notification_updates:
check_billable_units(update)
update_letter_notification(filename, temporary_failures, update)
sorted_letter_counts[update.cost_threshold] += 1
if temporary_failures:
# This will alert Notify that DVLA was unable to deliver the letters, we need to investigate
@@ -421,6 +426,32 @@ def update_letter_notifications_statuses(self, filename):
filename=filename, failures=temporary_failures)
raise DVLAException(message)
if sorted_letter_counts.keys() - {'Unsorted', 'Sorted'}:
unknown_status = sorted_letter_counts.keys() - {'Unsorted', 'Sorted'}
message = 'DVLA response file: {} contains unknown Sorted status {}'.format(
filename, unknown_status
)
raise DVLAException(message)
billing_date = get_billing_date_in_bst_from_filename(filename)
persist_daily_sorted_letter_counts(billing_date, sorted_letter_counts)
def get_billing_date_in_bst_from_filename(filename):
datetime_string = filename.split('.')[1]
datetime_obj = datetime.strptime(datetime_string, '%Y%m%d%H%M%S')
return convert_utc_to_bst(datetime_obj).date()
def persist_daily_sorted_letter_counts(day, sorted_letter_counts):
daily_letter_count = DailySortedLetter(
billing_day=day,
unsorted_count=sorted_letter_counts['Unsorted'],
sorted_count=sorted_letter_counts['Sorted']
)
dao_create_or_update_daily_sorted_letter(daily_letter_count)
def process_updates_from_file(response_file):
NotificationUpdate = namedtuple('NotificationUpdate', ['reference', 'status', 'page_count', 'cost_threshold'])
app/commands.py
+89 -8
View File
@@ -1,28 +1,33 @@
import functools
import uuid
from datetime import datetime, timedelta
from decimal import Decimal
import functools
import flask
from flask import current_app
import click
import flask
from click_datetime import Datetime as click_dt
from flask import current_app
from sqlalchemy.orm.exc import NoResultFound
from app import db
from app import db, DATETIME_FORMAT, encryption
from app.celery.scheduled_tasks import send_total_sent_notifications_to_performance_platform
from app.celery.service_callback_tasks import send_delivery_status_to_service
from app.config import QueueNames
from app.dao.monthly_billing_dao import (
create_or_update_monthly_billing,
get_monthly_billing_by_notification_type,
get_service_ids_that_need_billing_populated
)
from app.models import PROVIDERS, User, SMS_TYPE, EMAIL_TYPE
from app.dao.provider_rates_dao import create_provider_rates as dao_create_provider_rates
from app.dao.service_callback_api_dao import get_service_callback_api_for_service
from app.dao.services_dao import (
delete_service_and_all_associated_db_objects,
dao_fetch_all_services_by_user
)
from app.dao.provider_rates_dao import create_provider_rates as dao_create_provider_rates
from app.dao.users_dao import (delete_model_user, delete_user_verify_codes)
from app.models import PROVIDERS, User, SMS_TYPE, EMAIL_TYPE, Notification
from app.performance_platform.processing_time import (send_processing_time_for_start_and_end)
from app.utils import get_midnight_for_day_before, get_london_midnight_in_utc
from app.performance_platform.processing_time import send_processing_time_for_start_and_end
@click.group(name='command', help='Additional commands')
@@ -209,12 +214,38 @@ def populate_monthly_billing(year):
populate(service_id, year, i)
@notify_command()
@click.option('-s', '--start_date', required=True, help="start date inclusive", type=click_dt(format='%Y-%m-%d'))
@click.option('-e', '--end_date', required=True, help="end date inclusive", type=click_dt(format='%Y-%m-%d'))
def backfill_performance_platform_totals(start_date, end_date):
"""
Send historical total messages sent to Performance Platform.
WARNING: This does not overwrite existing data. You need to delete
the existing data or Performance Platform will double-count.
"""
delta = end_date - start_date
print('Sending total messages sent for all days between {} and {}'.format(start_date, end_date))
for i in range(delta.days + 1):
process_date = start_date + timedelta(days=i)
print('Sending total messages sent for {}'.format(
process_date.isoformat()
))
send_total_sent_notifications_to_performance_platform(process_date)
@notify_command()
@click.option('-s', '--start_date', required=True, help="start date inclusive", type=click_dt(format='%Y-%m-%d'))
@click.option('-e', '--end_date', required=True, help="end date inclusive", type=click_dt(format='%Y-%m-%d'))
def backfill_processing_time(start_date, end_date):
"""
Send historical performance platform stats.
Send historical processing time to Performance Platform.
"""
delta = end_date - start_date
@@ -284,5 +315,55 @@ def insert_inbound_numbers_from_file(file_name):
file.close()
@notify_command(name='replay-service-callbacks')
@click.option('-f', '--file_name', required=True,
help="""Full path of the file to upload, file is a contains client references of
notifications that need the status to be sent to the service.""")
@click.option('-s', '--service_id', required=True,
help="""The service that the callbacks are for""")
def replay_service_callbacks(file_name, service_id):
print("Start send service callbacks for service: ", service_id)
callback_api = get_service_callback_api_for_service(service_id=service_id)
if not callback_api:
print("Callback api was not found for service: {}".format(service_id))
return
errors = []
notifications = []
file = open(file_name)
for ref in file:
try:
notification = Notification.query.filter_by(client_reference=ref.strip()).one()
notifications.append(notification)
except NoResultFound as e:
errors.append("Reference: {} was not found in notifications.".format(ref))
for e in errors:
print(e)
if errors:
raise Exception("Some notifications for the given references were not found")
for n in notifications:
data = {
"notification_id": str(n.id),
"notification_client_reference": n.client_reference,
"notification_to": n.to,
"notification_status": n.status,
"notification_created_at": n.created_at.strftime(DATETIME_FORMAT),
"notification_updated_at": n.updated_at.strftime(DATETIME_FORMAT),
"notification_sent_at": n.sent_at.strftime(DATETIME_FORMAT),
"notification_type": n.notification_type,
"service_callback_api_url": callback_api.url,
"service_callback_api_bearer_token": callback_api.bearer_token,
}
encrypted_status_update = encryption.encrypt(data)
send_delivery_status_to_service.apply_async([str(n.id), encrypted_status_update],
queue=QueueNames.CALLBACKS)
print("Replay service status for service: {}. Sent {} notification status updates to the queue".format(
service_id, len(notifications)))
def setup_commands(application):
application.cli.add_command(command_group)
app/config.py
-5
View File
@@ -230,11 +230,6 @@ class Config(object):
'schedule': crontab(hour=4, minute=40),
'options': {'queue': QueueNames.PERIODIC}
},
'timeout-job-statistics': {
'task': 'timeout-job-statistics',
'schedule': crontab(hour=5, minute=0),
'options': {'queue': QueueNames.PERIODIC}
},
'populate_monthly_billing': {
'task': 'populate_monthly_billing',
'schedule': crontab(hour=5, minute=10),
app/dao/daily_sorted_letter_dao.py
+37
View File
@@ -0,0 +1,37 @@
from datetime import datetime
from sqlalchemy.dialects.postgresql import insert
from app import db
from app.dao.dao_utils import transactional
from app.models import DailySortedLetter
def dao_get_daily_sorted_letter_by_billing_day(billing_day):
return DailySortedLetter.query.filter_by(
billing_day=billing_day
).first()
@transactional
def dao_create_or_update_daily_sorted_letter(new_daily_sorted_letter):
'''
This uses the Postgres upsert to avoid race conditions when two threads try and insert
at the same row. The excluded object refers to values that we tried to insert but were
rejected.
http://docs.sqlalchemy.org/en/latest/dialects/postgresql.html#insert-on-conflict-upsert
'''
table = DailySortedLetter.__table__
stmt = insert(table).values(
billing_day=new_daily_sorted_letter.billing_day,
unsorted_count=new_daily_sorted_letter.unsorted_count,
sorted_count=new_daily_sorted_letter.sorted_count)
stmt = stmt.on_conflict_do_update(
index_elements=[table.c.billing_day],
set_={
'unsorted_count': table.c.unsorted_count + stmt.excluded.unsorted_count,
'sorted_count': table.c.sorted_count + stmt.excluded.sorted_count,
'updated_at': datetime.utcnow()
}
)
db.session.connection().execute(stmt)
app/dao/notifications_dao.py
+1
View File
@@ -83,6 +83,7 @@ def dao_get_template_usage(service_id, limit_days=None):
Template.id.label('template_id'),
Template.name,
Template.template_type,
Template.is_precompiled_letter,
notifications_aggregate_query.c.count
).join(
notifications_aggregate_query,
app/dao/services_dao.py
+4
View File
@@ -522,6 +522,7 @@ def dao_fetch_monthly_historical_usage_by_template_for_service(service_id, year)
stat.month = result.month
stat.year = result.year
stat.count = result.count
stat.is_precompiled_letter = result.is_precompiled_letter
stats.append(stat)
month = get_london_month_from_utc_column(Notification.created_at)
@@ -533,6 +534,7 @@ def dao_fetch_monthly_historical_usage_by_template_for_service(service_id, year)
if fy_start < datetime.now() < fy_end:
today_results = db.session.query(
Notification.template_id,
Template.is_precompiled_letter,
Template.name,
Template.template_type,
extract('month', month).label('month'),
@@ -547,6 +549,7 @@ def dao_fetch_monthly_historical_usage_by_template_for_service(service_id, year)
Notification.key_type != KEY_TYPE_TEST
).group_by(
Notification.template_id,
Template.hidden,
Template.name,
Template.template_type,
month,
@@ -571,6 +574,7 @@ def dao_fetch_monthly_historical_usage_by_template_for_service(service_id, year)
new_stat.month = int(today_result.month)
new_stat.year = int(today_result.year)
new_stat.count = today_result.count
new_stat.is_precompiled_letter = today_result.is_precompiled_letter
stats.append(new_stat)
return stats
app/dao/stats_template_usage_by_month_dao.py
+1
View File
@@ -37,6 +37,7 @@ def dao_get_template_usage_stats_by_service(service_id, year):
StatsTemplateUsageByMonth.template_id,
Template.name,
Template.template_type,
Template.is_precompiled_letter,
StatsTemplateUsageByMonth.month,
StatsTemplateUsageByMonth.year,
StatsTemplateUsageByMonth.count
app/dao/templates_dao.py
+6 -1
View File
@@ -5,7 +5,11 @@ from sqlalchemy import asc, desc
from sqlalchemy.sql.expression import bindparam
from app import db
from app.models import (Template, TemplateHistory, TemplateRedacted)
from app.models import (
Template,
TemplateHistory,
TemplateRedacted
)
from app.dao.dao_utils import (
transactional,
version_class
@@ -135,6 +139,7 @@ def dao_get_templates_for_cache(cache):
query = db.session.query(Template.id.label('template_id'),
Template.template_type,
Template.name,
Template.is_precompiled_letter,
cache_subq.c.count.label('count')
).join(cache_subq,
Template.id == cache_subq.c.template_id
app/delivery/send_to_providers.py
-5
View File
@@ -31,7 +31,6 @@ from app.models import (
NOTIFICATION_SENT,
NOTIFICATION_SENDING
)
from app.celery.statistics_tasks import create_initial_notification_statistic_tasks
def send_sms_to_provider(notification):
@@ -83,8 +82,6 @@ def send_sms_to_provider(notification):
notification.billable_units = template.fragment_count
update_notification(notification, provider, notification.international)
create_initial_notification_statistic_tasks(notification)
current_app.logger.debug(
"SMS {} sent to provider {} at {}".format(notification.id, provider.get_name(), notification.sent_at)
)
@@ -138,8 +135,6 @@ def send_email_to_provider(notification):
notification.reference = reference
update_notification(notification, provider)
create_initial_notification_statistic_tasks(notification)
current_app.logger.debug(
"Email {} sent to provider at {}".format(notification.id, notification.sent_at)
)
app/letters/utils.py
+28
View File
@@ -1,5 +1,6 @@
from datetime import datetime, timedelta
import boto3
from flask import current_app
from notifications_utils.s3 import s3upload
@@ -10,6 +11,8 @@ from app.variables import Retention
LETTERS_PDF_FILE_LOCATION_STRUCTURE = \
'{folder}/NOTIFY.{reference}.{duplex}.{letter_class}.{colour}.{crown}.{date}.pdf'
PRECOMPILED_BUCKET_PREFIX = '{folder}/NOTIFY.{reference}'
def get_letter_pdf_filename(reference, crown):
now = datetime.utcnow()
@@ -31,6 +34,15 @@ def get_letter_pdf_filename(reference, crown):
return upload_file_name
def get_bucket_prefix_for_notification(notification):
upload_file_name = PRECOMPILED_BUCKET_PREFIX.format(
folder=notification.created_at.date(),
reference=notification.reference
).upper()
return upload_file_name
def upload_letter_pdf(notification, pdf_data):
current_app.logger.info("PDF Letter {} reference {} created at {}, {} bytes".format(
notification.id, notification.reference, notification.created_at, len(pdf_data)))
@@ -48,3 +60,19 @@ def upload_letter_pdf(notification, pdf_data):
current_app.logger.info("Uploaded letters PDF {} to {} for notification id {}".format(
upload_file_name, current_app.config['LETTERS_PDF_BUCKET_NAME'], notification.id))
def get_letter_pdf(notification):
bucket_name = current_app.config['LETTERS_PDF_BUCKET_NAME']
s3 = boto3.resource('s3')
bucket = s3.Bucket(bucket_name)
for item in bucket.objects.filter(Prefix=get_bucket_prefix_for_notification(notification)):
obj = s3.Object(
bucket_name=bucket_name,
key=item.key
)
file_content = obj.get()["Body"].read()
return file_content
app/models.py
+24 -3
View File
@@ -6,6 +6,7 @@ from flask import url_for, current_app
from sqlalchemy.ext.declarative import declared_attr
from sqlalchemy.ext.associationproxy import association_proxy
from sqlalchemy.ext.hybrid import hybrid_property
from sqlalchemy.dialects.postgresql import (
UUID,
JSON
@@ -23,7 +24,7 @@ from notifications_utils.letter_timings import get_letter_timings
from notifications_utils.template import (
PlainTextEmailTemplate,
SMSMessageTemplate,
LetterDVLATemplate,
LetterPrintTemplate,
)
from app.encryption import (
@@ -641,6 +642,9 @@ class TemplateProcessTypes(db.Model):
name = db.Column(db.String(255), primary_key=True)
PRECOMPILED_TEMPLATE_NAME = 'Pre-compiled PDF'
class TemplateBase(db.Model):
__abstract__ = True
@@ -718,6 +722,14 @@ class TemplateBase(db.Model):
else:
return None
@hybrid_property
def is_precompiled_letter(self):
return self.hidden and self.name == PRECOMPILED_TEMPLATE_NAME and self.template_type == LETTER_TYPE
@is_precompiled_letter.setter
def is_precompiled_letter(self, value):
pass
def _as_utils_template(self):
if self.template_type == EMAIL_TYPE:
return PlainTextEmailTemplate(
@@ -728,9 +740,8 @@ class TemplateBase(db.Model):
{'content': self.content}
)
if self.template_type == LETTER_TYPE:
return LetterDVLATemplate(
return LetterPrintTemplate(
{'content': self.content, 'subject': self.subject},
notification_reference=1,
contact_block=self.service.get_default_letter_contact(),
)
@@ -1752,3 +1763,13 @@ class StatsTemplateUsageByMonth(db.Model):
'year': self.year,
'count': self.count
}
class DailySortedLetter(db.Model):
__tablename__ = "daily_sorted_letter"
id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
billing_day = db.Column(db.Date, nullable=False, index=True, unique=True)
unsorted_count = db.Column(db.Integer, nullable=False, default=0)
sorted_count = db.Column(db.Integer, nullable=False, default=0)
updated_at = db.Column(db.DateTime, nullable=True, onupdate=datetime.datetime.utcnow)
app/notifications/notifications_ses_callback.py
-2
View File
@@ -11,7 +11,6 @@ from app.dao import (
notifications_dao
)
from app.dao.service_callback_api_dao import get_service_callback_api_for_service
from app.celery.statistics_tasks import create_outcome_notification_statistic_tasks
from app.notifications.process_client_response import validate_callback_data
from app.celery.service_callback_tasks import send_delivery_status_to_service
from app.config import QueueNames
@@ -77,7 +76,6 @@ def process_ses_response(ses_request):
notification.sent_at
)
create_outcome_notification_statistic_tasks(notification)
_check_and_queue_callback_task(notification.id, notification.service_id)
return
app/notifications/process_client_response.py
-2
View File
@@ -8,7 +8,6 @@ from app.clients import ClientException
from app.dao import notifications_dao
from app.clients.sms.firetext import get_firetext_responses
from app.clients.sms.mmg import get_mmg_responses
from app.celery.statistics_tasks import create_outcome_notification_statistic_tasks
from app.celery.service_callback_tasks import send_delivery_status_to_service
from app.config import QueueNames
from app.dao.service_callback_api_dao import get_service_callback_api_for_service
@@ -80,7 +79,6 @@ def _process_for_status(notification_status, client_name, reference):
notification.sent_at
)
create_outcome_notification_statistic_tasks(notification)
# queue callback task only if the service_callback_api exists
service_callback_api = get_service_callback_api_for_service(service_id=notification.service_id)
app/organisation/rest.py
+23
View File
@@ -107,3 +107,26 @@ def get_organisation_users(organisation_id):
result = user_schema.dump(org_users, many=True)
return jsonify(data=result.data)
@organisation_blueprint.route('/unique', methods=["GET"])
def is_organisation_name_unique():
organisation_id, name = check_request_args(request)
name_exists = Organisation.query.filter(Organisation.name.ilike(name)).first()
result = (not name_exists) or str(name_exists.id) == organisation_id
return jsonify(result=result), 200
def check_request_args(request):
org_id = request.args.get('org_id')
name = request.args.get('name', None)
errors = []
if not org_id:
errors.append({'org_id': ["Can't be empty"]})
if not name:
errors.append({'name': ["Can't be empty"]})
if errors:
raise InvalidRequest(errors, status_code=400)
return org_id, name
app/performance_platform/total_sent_notifications.py
+10 -10
View File
@@ -1,11 +1,8 @@
from datetime import datetime
from datetime import timedelta
from app import performance_platform_client
from app.dao.notifications_dao import get_total_sent_notifications_in_date_range
from app.utils import (
get_london_midnight_in_utc,
get_midnight_for_day_before
)
from app.utils import get_london_midnight_in_utc
def send_total_notifications_sent_for_day_stats(date, notification_type, count):
@@ -20,13 +17,13 @@ def send_total_notifications_sent_for_day_stats(date, notification_type, count):
performance_platform_client.send_stats_to_performance_platform(payload)
def get_total_sent_notifications_yesterday():
today = datetime.utcnow()
start_date = get_midnight_for_day_before(today)
end_date = get_london_midnight_in_utc(today)
def get_total_sent_notifications_for_day(day):
start_date = get_london_midnight_in_utc(day)
end_date = start_date + timedelta(days=1)
email_count = get_total_sent_notifications_in_date_range(start_date, end_date, 'email')
sms_count = get_total_sent_notifications_in_date_range(start_date, end_date, 'sms')
letter_count = get_total_sent_notifications_in_date_range(start_date, end_date, 'letter')
return {
"start_date": start_date,
@@ -35,5 +32,8 @@ def get_total_sent_notifications_yesterday():
},
"sms": {
"count": sms_count
}
},
"letter": {
"count": letter_count
},
}
app/schemas.py
+10 -1
View File
@@ -454,7 +454,16 @@ class NotificationWithTemplateSchema(BaseSchema):
template = fields.Nested(
TemplateSchema,
only=['id', 'version', 'name', 'template_type', 'content', 'subject', 'redact_personalisation'],
only=[
'id',
'version',
'name',
'template_type',
'content',
'subject',
'redact_personalisation',
'is_precompiled_letter'
],
dump_only=True
)
job = fields.Nested(JobSchema, only=["id", "original_file_name"], dump_only=True)
app/service/rest.py
+2 -1
View File
@@ -536,7 +536,8 @@ def get_monthly_template_usage(service_id):
'type': i.template_type,
'month': i.month,
'year': i.year,
'count': i.count
'count': i.count,
'is_precompiled_letter': i.is_precompiled_letter
}
)
app/template/rest.py
+75 -25
View File
@@ -1,4 +1,6 @@
import base64
import botocore
from flask import (
Blueprint,
current_app,
@@ -18,6 +20,7 @@ from app.dao.templates_dao import (
dao_get_template_by_id)
from notifications_utils.template import SMSMessageTemplate
from app.dao.services_dao import dao_fetch_service_by_id
from app.letters.utils import get_letter_pdf
from app.models import SMS_TYPE
from app.notifications.validators import service_has_permission, check_reply_to
from app.schemas import (template_schema, template_history_schema)
@@ -29,7 +32,6 @@ from app.utils import get_template_instance, get_public_notify_type_text
template_blueprint = Blueprint('template', __name__, url_prefix='/service/<uuid:service_id>/template')
register_errors(template_blueprint)
@@ -194,36 +196,84 @@ def preview_letter_template_by_notification_id(service_id, notification_id, file
template = dao_get_template_by_id(notification.template_id)
template_for_letter_print = {
"id": str(notification.template_id),
"subject": template.subject,
"content": template.content,
"version": str(template.version)
}
if template.is_precompiled_letter:
service = dao_fetch_service_by_id(service_id)
try:
data = {
'letter_contact_block': notification.reply_to_text,
'template': template_for_letter_print,
'values': notification.personalisation,
'dvla_org_id': service.dvla_organisation_id,
}
pdf_file = get_letter_pdf(notification)
resp = requests_post(
'{}/preview.{}{}'.format(
except botocore.exceptions.ClientError:
current_app.logger.exception(
'Error getting letter file from S3 notification id {}'.format(notification_id))
raise InvalidRequest('Error getting letter file from S3 notification id {}'.format(notification_id),
status_code=500)
content = base64.b64encode(pdf_file).decode('utf-8')
if file_type == 'png':
url = '{}/precompiled-preview.png{}'.format(
current_app.config['TEMPLATE_PREVIEW_API_HOST'],
'?page={}'.format(page) if page else ''
)
content = _get_png_preview(url, content, notification.id, json=False)
else:
template_for_letter_print = {
"id": str(notification.template_id),
"subject": template.subject,
"content": template.content,
"version": str(template.version)
}
service = dao_fetch_service_by_id(service_id)
data = {
'letter_contact_block': notification.reply_to_text,
'template': template_for_letter_print,
'values': notification.personalisation,
'dvla_org_id': service.dvla_organisation_id,
}
url = '{}/preview.{}{}'.format(
current_app.config['TEMPLATE_PREVIEW_API_HOST'],
file_type,
'?page={}'.format(page) if page else ''
),
json=data,
headers={'Authorization': 'Token {}'.format(current_app.config['TEMPLATE_PREVIEW_API_KEY'])}
)
if resp.status_code != 200:
raise InvalidRequest(
'Error generating preview for {}'.format(notification_id), status_code=500
)
content = base64.b64encode(resp.content).decode('utf-8')
content = _get_png_preview(url, data, notification.id, json=True)
return jsonify({"content": content})
def _get_png_preview(url, data, notification_id, json=True):
if json:
resp = requests_post(
url,
json=data,
headers={'Authorization': 'Token {}'.format(current_app.config['TEMPLATE_PREVIEW_API_KEY'])}
)
else:
resp = requests_post(
url,
data=data,
headers={'Authorization': 'Token {}'.format(current_app.config['TEMPLATE_PREVIEW_API_KEY'])}
)
if resp.status_code != 200:
current_app.logger.exception(
'Error generating preview letter for {} \nStatus code: {}\n{}'.format(
notification_id,
resp.status_code,
resp.content
))
raise InvalidRequest(
'Error generating preview letter for {}\nStatus code: {}\n{}'.format(
notification_id,
resp.status_code,
resp.content
), status_code=500
)
return base64.b64encode(resp.content).decode('utf-8')
app/template_statistics/rest.py
+2 -1
View File
@@ -47,7 +47,8 @@ def get_template_statistics_for_service_by_day(service_id):
'count': data.count,
'template_id': str(data.template_id),
'template_name': data.name,
'template_type': data.template_type
'template_type': data.template_type,
'is_precompiled_letter': data.is_precompiled_letter
}
return jsonify(data=[serialize(row) for row in stats])
docker/Dockerfile
+2
View File
@@ -19,6 +19,8 @@ RUN \
build-essential \
zip \
libpq-dev \
libffi-dev \
python-dev \
jq \
&& echo "Clean up" \
&& rm -rf /var/lib/apt/lists/* /tmp/*
manifest-delivery-base.yml
+1 -1
View File
@@ -95,6 +95,6 @@ applications:
NOTIFY_APP_NAME: delivery-worker-receipts
- name: notify-delivery-worker-service-callbacks
command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO -P eventlet -c 1000 -Q service-callbacks
command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q service-callbacks
env:
NOTIFY_APP_NAME: delivery-worker-service-callbacks
migrations/versions/0173_create_daily_sorted_letter.py
+30
View File
@@ -0,0 +1,30 @@
"""
Revision ID: 0173_create_daily_sorted_letter
Revises: 0172_deprioritise_examples
Create Date: 2018-03-01 11:53:32.964256
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
revision = '0173_create_daily_sorted_letter'
down_revision = '0172_deprioritise_examples'
def upgrade():
op.create_table('daily_sorted_letter',
sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False),
sa.Column('billing_day', sa.Date(), nullable=False),
sa.Column('unsorted_count', sa.Integer(), nullable=False),
sa.Column('sorted_count', sa.Integer(), nullable=False),
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.PrimaryKeyConstraint('id')
)
op.create_index(op.f('ix_daily_sorted_letter_billing_day'), 'daily_sorted_letter', ['billing_day'], unique=True)
def downgrade():
op.drop_index(op.f('ix_daily_sorted_letter_billing_day'), table_name='daily_sorted_letter')
op.drop_table('daily_sorted_letter')
requirements.txt
+2 -2
View File
@@ -16,13 +16,13 @@ marshmallow==2.15.0
monotonic==1.4
psycopg2-binary==2.7.4
PyJWT==1.6.0
SQLAlchemy==1.2.4
SQLAlchemy==1.2.5
notifications-python-client==4.7.2
# PaaS
awscli-cwlogs>=1.4,<1.5
git+https://github.com/alphagov/notifications-utils.git@23.8.0#egg=notifications-utils==23.8.0
git+https://github.com/alphagov/notifications-utils.git@24.0.0#egg=notifications-utils==24.0.0
git+https://github.com/alphagov/boto.git@2.43.0-patch3#egg=boto==2.43.0-patch3
requirements_for_test.txt
+4 -4
View File
@@ -1,12 +1,12 @@
-r requirements.txt
flake8==3.5.0
pytest==3.4.1
pytest==3.4.2
pytest-env==0.6.2
pytest-mock==1.7.0
pytest-mock==1.7.1
pytest-cov==2.5.1
pytest-xdist==1.22.2
coveralls==1.2.0
freezegun==0.3.9
coveralls==1.3.0
freezegun==0.3.10
requests-mock==1.4.0
# optional requirements for jsonschema
strict-rfc3339==0.7
tests/app/celery/test_ftp_update_tasks.py
+96 -11
View File
@@ -1,5 +1,5 @@
from collections import namedtuple
from datetime import datetime
from collections import namedtuple, defaultdict
from datetime import datetime, date
import pytest
from freezegun import freeze_time
@@ -18,12 +18,15 @@ from app.models import (
)
from app.celery.tasks import (
check_billable_units,
get_billing_date_in_bst_from_filename,
persist_daily_sorted_letter_counts,
process_updates_from_file,
update_dvla_job_to_error,
update_letter_notifications_statuses,
update_letter_notifications_to_error,
update_letter_notifications_to_sent_to_dvla
)
from app.dao.daily_sorted_letter_dao import dao_get_daily_sorted_letter_by_billing_day
from tests.app.db import create_notification, create_service_callback_api
from tests.conftest import set_config
@@ -56,8 +59,8 @@ def test_update_letter_notifications_statuses_raises_for_invalid_format(notify_a
mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=invalid_file)
with pytest.raises(DVLAException) as e:
update_letter_notifications_statuses(filename='foo.txt')
assert 'DVLA response file: {} has an invalid format'.format('foo.txt') in str(e)
update_letter_notifications_statuses(filename='NOTIFY.20170823160812.RSP.TXT')
assert 'DVLA response file: {} has an invalid format'.format('NOTIFY.20170823160812.RSP.TXT') in str(e)
def test_update_letter_notification_statuses_when_notification_does_not_exist_updates_notification_history(
@@ -70,7 +73,7 @@ def test_update_letter_notification_statuses_when_notification_does_not_exist_up
billable_units=1)
Notification.query.filter_by(id=notification.id).delete()
update_letter_notifications_statuses(filename="older_than_7_days.txt")
update_letter_notifications_statuses(filename="NOTIFY.20170823160812.RSP.TXT")
updated_history = NotificationHistory.query.filter_by(id=notification.id).one()
assert updated_history.status == NOTIFICATION_DELIVERED
@@ -90,12 +93,35 @@ def test_update_letter_notifications_statuses_raises_dvla_exception(notify_api,
) in str(e)
def test_update_letter_notifications_statuses_raises_error_for_unknown_sorted_status(
notify_api,
mocker,
sample_letter_template
):
sent_letter_1 = create_notification(sample_letter_template, reference='ref-foo', status=NOTIFICATION_SENDING)
sent_letter_2 = create_notification(sample_letter_template, reference='ref-bar', status=NOTIFICATION_SENDING)
valid_file = '{}|Sent|1|Unsorted\n{}|Sent|2|Error'.format(
sent_letter_1.reference, sent_letter_2.reference)
mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=valid_file)
with pytest.raises(DVLAException) as e:
update_letter_notifications_statuses(filename='NOTIFY.20170823160812.RSP.TXT')
assert "DVLA response file: {filename} contains unknown Sorted status {unknown_status}".format(
filename="NOTIFY.20170823160812.RSP.TXT", unknown_status="{'Error'}"
) in str(e)
def test_update_letter_notifications_statuses_calls_with_correct_bucket_location(notify_api, mocker):
s3_mock = mocker.patch('app.celery.tasks.s3.get_s3_object')
with set_config(notify_api, 'NOTIFY_EMAIL_DOMAIN', 'foo.bar'):
update_letter_notifications_statuses(filename='foo.txt')
s3_mock.assert_called_with('{}-ftp'.format(current_app.config['NOTIFY_EMAIL_DOMAIN']), 'foo.txt')
update_letter_notifications_statuses(filename='NOTIFY.20170823160812.RSP.TXT')
s3_mock.assert_called_with('{}-ftp'.format(
current_app.config['NOTIFY_EMAIL_DOMAIN']),
'NOTIFY.20170823160812.RSP.TXT'
)
def test_update_letter_notifications_statuses_builds_updates_from_content(notify_api, mocker):
@@ -103,7 +129,7 @@ def test_update_letter_notifications_statuses_builds_updates_from_content(notify
mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=valid_file)
update_mock = mocker.patch('app.celery.tasks.process_updates_from_file')
update_letter_notifications_statuses(filename='foo.txt')
update_letter_notifications_statuses(filename='NOTIFY.20170823160812.RSP.TXT')
update_mock.assert_called_with('ref-foo|Sent|1|Unsorted\nref-bar|Sent|2|Sorted')
@@ -136,7 +162,7 @@ def test_update_letter_notifications_statuses_persisted(notify_api, mocker, samp
mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=valid_file)
with pytest.raises(expected_exception=DVLAException) as e:
update_letter_notifications_statuses(filename='foo.txt')
update_letter_notifications_statuses(filename='NOTIFY.20170823160812.RSP.TXT')
assert sent_letter.status == NOTIFICATION_DELIVERED
assert sent_letter.billable_units == 1
@@ -145,7 +171,45 @@ def test_update_letter_notifications_statuses_persisted(notify_api, mocker, samp
assert failed_letter.billable_units == 2
assert failed_letter.updated_at
assert "DVLA response file: {filename} has failed letters with notification.reference {failures}".format(
filename="foo.txt", failures=[format(failed_letter.reference)]) in str(e)
filename="NOTIFY.20170823160812.RSP.TXT", failures=[format(failed_letter.reference)]) in str(e)
def test_update_letter_notifications_statuses_persists_daily_sorted_letter_count(
notify_api,
mocker,
sample_letter_template
):
sent_letter_1 = create_notification(sample_letter_template, reference='ref-foo', status=NOTIFICATION_SENDING)
sent_letter_2 = create_notification(sample_letter_template, reference='ref-bar', status=NOTIFICATION_SENDING)
valid_file = '{}|Sent|1|Unsorted\n{}|Sent|2|Sorted'.format(
sent_letter_1.reference, sent_letter_2.reference)
mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=valid_file)
persist_letter_count_mock = mocker.patch('app.celery.tasks.persist_daily_sorted_letter_counts')
update_letter_notifications_statuses(filename='NOTIFY.20170823160812.RSP.TXT')
persist_letter_count_mock.assert_called_once_with(date(2017, 8, 23), {'Unsorted': 1, 'Sorted': 1})
def test_update_letter_notifications_statuses_persists_daily_sorted_letter_count_with_no_sorted_values(
notify_api,
mocker,
sample_letter_template,
notify_db_session
):
sent_letter_1 = create_notification(sample_letter_template, reference='ref-foo', status=NOTIFICATION_SENDING)
sent_letter_2 = create_notification(sample_letter_template, reference='ref-bar', status=NOTIFICATION_SENDING)
valid_file = '{}|Sent|1|Unsorted\n{}|Sent|2|Unsorted'.format(
sent_letter_1.reference, sent_letter_2.reference)
mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=valid_file)
update_letter_notifications_statuses(filename='NOTIFY.20170823160812.RSP.TXT')
daily_sorted_letter = dao_get_daily_sorted_letter_by_billing_day(date(2017, 8, 23))
assert daily_sorted_letter.unsorted_count == 2
assert daily_sorted_letter.sorted_count == 0
def test_update_letter_notifications_does_not_call_send_callback_if_no_db_entry(notify_api, mocker,
@@ -159,7 +223,7 @@ def test_update_letter_notifications_does_not_call_send_callback_if_no_db_entry(
'app.celery.service_callback_tasks.send_delivery_status_to_service.apply_async'
)
update_letter_notifications_statuses(filename='foo.txt')
update_letter_notifications_statuses(filename='NOTIFY.20170823160812.RSP.TXT')
send_mock.assert_not_called()
@@ -230,3 +294,24 @@ def test_check_billable_units_when_billable_units_does_not_match_page_count(
mock_logger.assert_called_once_with(
'Notification with id {} had 3 billable_units but a page count of 1'.format(notification.id)
)
@pytest.mark.parametrize('filename_date, billing_date', [
('20170820230000', date(2017, 8, 21)),
('20170120230000', date(2017, 1, 20))
])
def test_get_billing_date_in_bst_from_filename(filename_date, billing_date):
filename = 'NOTIFY.{}.RSP.TXT'.format(filename_date)
result = get_billing_date_in_bst_from_filename(filename)
assert result == billing_date
@freeze_time("2018-01-11 09:00:00")
def test_persist_daily_sorted_letter_counts_saves_sorted_and_unsorted_values(client, notify_db_session):
letter_counts = defaultdict(int, **{'Unsorted': 5, 'Sorted': 1})
persist_daily_sorted_letter_counts(date.today(), letter_counts)
day = dao_get_daily_sorted_letter_by_billing_day(date.today())
assert day.unsorted_count == 5
assert day.sorted_count == 1
tests/app/celery/test_scheduled_tasks.py
+1 -1
View File
@@ -335,7 +335,7 @@ def test_send_total_sent_notifications_to_performance_platform_calls_with_correc
new_callable=PropertyMock
) as mock_active:
mock_active.return_value = True
send_total_sent_notifications_to_performance_platform()
send_total_sent_notifications_to_performance_platform(yesterday)
perf_mock.assert_has_calls([
call(get_london_midnight_in_utc(yesterday), 'sms', 2),
tests/app/celery/test_service_callback_tasks.py
+151 -56
View File
@@ -1,39 +1,147 @@
import uuid
import json
from datetime import datetime
from requests import RequestException
import pytest
import requests_mock
from sqlalchemy.exc import SQLAlchemyError
from requests import RequestException
from app import (DATETIME_FORMAT, encryption)
from app import (DATETIME_FORMAT)
from tests.app.conftest import (
sample_service as create_sample_service,
sample_template as create_sample_template,
)
from tests.app.db import (
create_notification,
create_user,
create_service_callback_api
create_service_callback_api,
create_service,
create_template
)
from app.celery.service_callback_tasks import send_delivery_status_to_service
from app.config import QueueNames
@pytest.mark.parametrize("notification_type",
["email", "letter", "sms"])
def test_send_delivery_status_to_service_post_https_request_to_service(notify_db,
notify_db_session,
notification_type):
user = create_user()
service = create_sample_service(notify_db, notify_db_session, user=user, restricted=True)
def test_send_delivery_status_to_service_post_https_request_to_service_with_encrypted_data(
notify_db_session, notification_type):
callback_api, template = _set_up_test_data(notification_type)
datestr = datetime(2017, 6, 20)
notification = create_notification(template=template,
created_at=datestr,
updated_at=datestr,
sent_at=datestr,
status='sent'
)
encrypted_status_update = _set_up_encrypted_data(callback_api, notification)
with requests_mock.Mocker() as request_mock:
request_mock.post(callback_api.url,
json={},
status_code=200)
send_delivery_status_to_service(notification.id, encrypted_status_update=encrypted_status_update)
mock_data = {
"id": str(notification.id),
"reference": notification.client_reference,
"to": notification.to,
"status": notification.status,
"created_at": datestr.strftime(DATETIME_FORMAT),
"completed_at": datestr.strftime(DATETIME_FORMAT),
"sent_at": datestr.strftime(DATETIME_FORMAT),
"notification_type": notification_type
}
assert request_mock.call_count == 1
assert request_mock.request_history[0].url == callback_api.url
assert request_mock.request_history[0].method == 'POST'
assert request_mock.request_history[0].text == json.dumps(mock_data)
assert request_mock.request_history[0].headers["Content-type"] == "application/json"
assert request_mock.request_history[0].headers["Authorization"] == "Bearer {}".format(callback_api.bearer_token)
@pytest.mark.parametrize("notification_type",
["email", "letter", "sms"])
def test_send_delivery_status_to_service_retries_if_request_returns_500_with_encrypted_data(
notify_db_session, mocker, notification_type
):
callback_api, template = _set_up_test_data(notification_type)
datestr = datetime(2017, 6, 20)
notification = create_notification(template=template,
created_at=datestr,
updated_at=datestr,
sent_at=datestr,
status='sent'
)
encrypted_data = _set_up_encrypted_data(callback_api, notification)
mocked = mocker.patch('app.celery.service_callback_tasks.send_delivery_status_to_service.retry')
with requests_mock.Mocker() as request_mock:
request_mock.post(callback_api.url,
json={},
status_code=500)
send_delivery_status_to_service(notification.id, encrypted_status_update=encrypted_data)
assert mocked.call_count == 1
assert mocked.call_args[1]['queue'] == 'retry-tasks'
@pytest.mark.parametrize("notification_type",
["email", "letter", "sms"])
def test_send_delivery_status_to_service_does_not_retries_if_request_returns_404_with_encrypted_data(
notify_db_session,
mocker,
notification_type
):
callback_api, template = _set_up_test_data(notification_type)
datestr = datetime(2017, 6, 20)
notification = create_notification(template=template,
created_at=datestr,
updated_at=datestr,
sent_at=datestr,
status='sent'
)
encrypted_data = _set_up_encrypted_data(callback_api, notification)
mocked = mocker.patch('app.celery.service_callback_tasks.send_delivery_status_to_service.retry')
with requests_mock.Mocker() as request_mock:
request_mock.post(callback_api.url,
json={},
status_code=404)
send_delivery_status_to_service(notification.id, encrypted_status_update=encrypted_data)
assert mocked.call_count == 0
def _set_up_test_data(notification_type):
service = create_service(restricted=True)
template = create_template(service=service, template_type=notification_type, subject='Hello')
callback_api = create_service_callback_api(service=service, url="https://some.service.gov.uk/",
bearer_token="something_unique")
template = create_sample_template(
notify_db, notify_db_session, service=service, template_type=notification_type, subject_line='Hello'
)
return callback_api, template
def _set_up_encrypted_data(callback_api, notification):
data = {
"notification_id": str(notification.id),
"notification_client_reference": notification.client_reference,
"notification_to": notification.to,
"notification_status": notification.status,
"notification_created_at": notification.created_at.strftime(DATETIME_FORMAT),
"notification_updated_at": notification.updated_at.strftime(DATETIME_FORMAT),
"notification_sent_at": notification.sent_at.strftime(DATETIME_FORMAT),
"notification_type": notification.notification_type,
"service_callback_api_url": callback_api.url,
"service_callback_api_bearer_token": callback_api.bearer_token,
}
encrypted_status_update = encryption.encrypt(data)
return encrypted_status_update
# We are updating the task to take everything it needs so that there are no db calls.
# The following tests will be deleted once that is complete.
@pytest.mark.parametrize("notification_type",
["email", "letter", "sms"])
def test_send_delivery_status_to_service_post_https_request_to_service(
notify_db_session, notification_type):
callback_api, template = _set_up_test_data(notification_type)
datestr = datetime(2017, 6, 20)
notification = create_notification(template=template,
@@ -71,12 +179,9 @@ def test_send_delivery_status_to_service_post_https_request_to_service(notify_db
@pytest.mark.parametrize("notification_type",
["email", "letter", "sms"])
def test_send_delivery_status_to_service_does_not_sent_request_when_service_callback_api_does_not_exist(
notify_db, notify_db_session, mocker, notification_type):
service = create_sample_service(notify_db, notify_db_session, restricted=True)
template = create_sample_template(
notify_db, notify_db_session, service=service, template_type=notification_type, subject_line='Hello'
)
notify_db_session, mocker, notification_type):
service = create_service(restricted=True)
template = create_template(service=service, template_type=notification_type, subject='Hello')
datestr = datetime(2017, 6, 20)
notification = create_notification(template=template,
@@ -88,23 +193,15 @@ def test_send_delivery_status_to_service_does_not_sent_request_when_service_call
mocked = mocker.patch("requests.request")
send_delivery_status_to_service(notification.id)
mocked.call_count == 0
assert mocked.call_count == 0
@pytest.mark.parametrize("notification_type",
["email", "letter", "sms"])
def test_send_delivery_status_to_service_retries_if_request_returns_500(notify_db,
notify_db_session,
def test_send_delivery_status_to_service_retries_if_request_returns_500(notify_db_session,
mocker,
notification_type):
user = create_user()
service = create_sample_service(notify_db, notify_db_session, user=user, restricted=True)
template = create_sample_template(
notify_db, notify_db_session, service=service, template_type=notification_type, subject_line='Hello'
)
callback_api = create_service_callback_api(service=service, url="https://some.service.gov.uk/",
bearer_token="something_unique")
callback_api, template = _set_up_test_data(notification_type)
datestr = datetime(2017, 6, 20)
notification = create_notification(template=template,
created_at=datestr,
@@ -125,18 +222,11 @@ def test_send_delivery_status_to_service_retries_if_request_returns_500(notify_d
@pytest.mark.parametrize("notification_type",
["email", "letter", "sms"])
def test_send_delivery_status_to_service_retries_if_request_throws_unknown(notify_db,
notify_db_session,
def test_send_delivery_status_to_service_retries_if_request_throws_unknown(notify_db_session,
mocker,
notification_type):
user = create_user()
service = create_sample_service(notify_db, notify_db_session, user=user, restricted=True)
template = create_sample_template(
notify_db, notify_db_session, service=service, template_type=notification_type, subject_line='Hello'
)
create_service_callback_api(service=service, url="https://some.service.gov.uk/",
bearer_token="something_unique")
callback_api, template = _set_up_test_data(notification_type)
datestr = datetime(2017, 6, 20)
notification = create_notification(template=template,
created_at=datestr,
@@ -156,18 +246,12 @@ def test_send_delivery_status_to_service_retries_if_request_throws_unknown(notif
@pytest.mark.parametrize("notification_type",
["email", "letter", "sms"])
def test_send_delivery_status_to_service_does_not_retries_if_request_returns_404(notify_db,
notify_db_session,
mocker,
notification_type):
user = create_user()
service = create_sample_service(notify_db, notify_db_session, user=user, restricted=True)
template = create_sample_template(
notify_db, notify_db_session, service=service, template_type=notification_type, subject_line='Hello'
)
callback_api = create_service_callback_api(service=service, url="https://some.service.gov.uk/",
bearer_token="something_unique")
def test_send_delivery_status_to_service_does_not_retries_if_request_returns_404(
notify_db_session,
mocker,
notification_type
):
callback_api, template = _set_up_test_data(notification_type)
datestr = datetime(2017, 6, 20)
notification = create_notification(template=template,
created_at=datestr,
@@ -182,4 +266,15 @@ def test_send_delivery_status_to_service_does_not_retries_if_request_returns_404
status_code=404)
send_delivery_status_to_service(notification.id)
mocked.call_count == 0
assert mocked.call_count == 0
def test_send_delivery_status_to_service_retries_if_database_error(client, mocker):
notification_id = uuid.uuid4()
db_call = mocker.patch('app.celery.service_callback_tasks.get_notification_by_id', side_effect=SQLAlchemyError)
retry = mocker.patch('app.celery.service_callback_tasks.send_delivery_status_to_service.retry')
send_delivery_status_to_service(notification_id)
db_call.assert_called_once_with(notification_id)
retry.assert_called_once_with(queue=QueueNames.RETRY)
tests/app/celery/test_statistics_tasks.py
-169
View File
@@ -1,169 +0,0 @@
import pytest
from app.celery.statistics_tasks import (
record_initial_job_statistics,
record_outcome_job_statistics,
create_initial_notification_statistic_tasks,
create_outcome_notification_statistic_tasks)
from sqlalchemy.exc import SQLAlchemyError
from app import create_uuid
from tests.app.conftest import sample_notification
from app.models import (
NOTIFICATION_STATUS_TYPES_COMPLETED,
NOTIFICATION_SENDING,
NOTIFICATION_PENDING,
NOTIFICATION_CREATED,
NOTIFICATION_DELIVERED,
)
def test_should_create_initial_job_task_if_notification_is_related_to_a_job(
notify_db, notify_db_session, sample_job, mocker
):
mock = mocker.patch("app.celery.statistics_tasks.record_initial_job_statistics.apply_async")
notification = sample_notification(notify_db, notify_db_session, job=sample_job)
create_initial_notification_statistic_tasks(notification)
mock.assert_called_once_with((str(notification.id), ), queue="statistics-tasks")
@pytest.mark.parametrize('status', [
NOTIFICATION_SENDING, NOTIFICATION_CREATED, NOTIFICATION_PENDING
])
def test_should_create_intial_job_task_if_notification_is_not_in_completed_state(
notify_db, notify_db_session, sample_job, mocker, status
):
mock = mocker.patch("app.celery.statistics_tasks.record_initial_job_statistics.apply_async")
notification = sample_notification(notify_db, notify_db_session, job=sample_job, status=status)
create_initial_notification_statistic_tasks(notification)
mock.assert_called_once_with((str(notification.id), ), queue="statistics-tasks")
def test_should_not_create_initial_job_task_if_notification_is_not_related_to_a_job(
notify_db, notify_db_session, mocker
):
notification = sample_notification(notify_db, notify_db_session, status=NOTIFICATION_CREATED)
mock = mocker.patch("app.celery.statistics_tasks.record_initial_job_statistics.apply_async")
create_initial_notification_statistic_tasks(notification)
mock.assert_not_called()
def test_should_create_outcome_job_task_if_notification_is_related_to_a_job(
notify_db, notify_db_session, sample_job, mocker
):
mock = mocker.patch("app.celery.statistics_tasks.record_outcome_job_statistics.apply_async")
notification = sample_notification(notify_db, notify_db_session, job=sample_job, status=NOTIFICATION_DELIVERED)
create_outcome_notification_statistic_tasks(notification)
mock.assert_called_once_with((str(notification.id), ), queue="statistics-tasks")
@pytest.mark.parametrize('status', NOTIFICATION_STATUS_TYPES_COMPLETED)
def test_should_create_outcome_job_task_if_notification_is_in_completed_state(
notify_db, notify_db_session, sample_job, mocker, status
):
mock = mocker.patch("app.celery.statistics_tasks.record_outcome_job_statistics.apply_async")
notification = sample_notification(notify_db, notify_db_session, job=sample_job, status=status)
create_outcome_notification_statistic_tasks(notification)
mock.assert_called_once_with((str(notification.id), ), queue="statistics-tasks")
@pytest.mark.parametrize('status', [
NOTIFICATION_SENDING, NOTIFICATION_CREATED, NOTIFICATION_PENDING
])
def test_should_not_create_outcome_job_task_if_notification_is_not_in_completed_state_already(
notify_db, notify_db_session, sample_job, mocker, status
):
mock = mocker.patch("app.celery.statistics_tasks.record_initial_job_statistics.apply_async")
notification = sample_notification(notify_db, notify_db_session, job=sample_job, status=status)
create_outcome_notification_statistic_tasks(notification)
mock.assert_not_called()
def test_should_not_create_outcome_job_task_if_notification_is_not_related_to_a_job(
notify_db, notify_db_session, sample_notification, mocker
):
mock = mocker.patch("app.celery.statistics_tasks.record_outcome_job_statistics.apply_async")
create_outcome_notification_statistic_tasks(sample_notification)
mock.assert_not_called()
def test_should_call_create_job_stats_dao_methods(notify_db, notify_db_session, sample_notification, mocker):
dao_mock = mocker.patch("app.celery.statistics_tasks.create_or_update_job_sending_statistics")
record_initial_job_statistics(str(sample_notification.id))
dao_mock.assert_called_once_with(sample_notification)
def test_should_retry_if_persisting_the_job_stats_has_a_sql_alchemy_exception(
notify_db,
notify_db_session,
sample_notification,
mocker):
dao_mock = mocker.patch(
"app.celery.statistics_tasks.create_or_update_job_sending_statistics",
side_effect=SQLAlchemyError()
)
retry_mock = mocker.patch('app.celery.statistics_tasks.record_initial_job_statistics.retry')
record_initial_job_statistics(str(sample_notification.id))
dao_mock.assert_called_once_with(sample_notification)
retry_mock.assert_called_with(queue="retry-tasks")
def test_should_call_update_job_stats_dao_outcome_methods(notify_db, notify_db_session, sample_notification, mocker):
dao_mock = mocker.patch("app.celery.statistics_tasks.update_job_stats_outcome_count")
record_outcome_job_statistics(str(sample_notification.id))
dao_mock.assert_called_once_with(sample_notification)
def test_should_retry_if_persisting_the_job_outcome_stats_has_a_sql_alchemy_exception(
notify_db,
notify_db_session,
sample_notification,
mocker):
dao_mock = mocker.patch(
"app.celery.statistics_tasks.update_job_stats_outcome_count",
side_effect=SQLAlchemyError()
)
retry_mock = mocker.patch('app.celery.statistics_tasks.record_outcome_job_statistics.retry')
record_outcome_job_statistics(str(sample_notification.id))
dao_mock.assert_called_once_with(sample_notification)
retry_mock.assert_called_with(queue="retry-tasks")
def test_should_retry_if_persisting_the_job_outcome_stats_updates_zero_rows(
notify_db,
notify_db_session,
sample_notification,
mocker):
dao_mock = mocker.patch("app.celery.statistics_tasks.update_job_stats_outcome_count", return_value=0)
retry_mock = mocker.patch('app.celery.statistics_tasks.record_outcome_job_statistics.retry')
record_outcome_job_statistics(str(sample_notification.id))
dao_mock.assert_called_once_with(sample_notification)
retry_mock.assert_called_with(queue="retry-tasks")
def test_should_retry_if_persisting_the_job_stats_creation_cant_find_notification_by_id(
notify_db,
notify_db_session,
mocker):
dao_mock = mocker.patch("app.celery.statistics_tasks.create_or_update_job_sending_statistics")
retry_mock = mocker.patch('app.celery.statistics_tasks.record_initial_job_statistics.retry')
record_initial_job_statistics(str(create_uuid()))
dao_mock.assert_not_called()
retry_mock.assert_called_with(queue="retry-tasks")
def test_should_retry_if_persisting_the_job_stats_outcome_cant_find_notification_by_id(
notify_db,
notify_db_session,
mocker):
dao_mock = mocker.patch("app.celery.statistics_tasks.update_job_stats_outcome_count")
retry_mock = mocker.patch('app.celery.statistics_tasks.record_outcome_job_statistics.retry')
record_outcome_job_statistics(str(create_uuid()))
dao_mock.assert_not_called()
retry_mock.assert_called_with(queue="retry-tasks")
tests/app/celery/test_tasks.py
+1 -9
View File
@@ -9,7 +9,7 @@ from freezegun import freeze_time
from requests import RequestException
from sqlalchemy.exc import SQLAlchemyError
from celery.exceptions import Retry
from notifications_utils.template import SMSMessageTemplate, WithSubjectTemplate, LetterDVLATemplate
from notifications_utils.template import SMSMessageTemplate, WithSubjectTemplate
from app import (encryption, DATETIME_FORMAT)
from app.celery import provider_tasks
@@ -1209,14 +1209,6 @@ def test_get_template_class(template_type, expected_class):
assert get_template_class(template_type) == expected_class
@freeze_time("2017-03-23 11:09:00.061258")
def test_dvla_letter_template(sample_letter_notification):
t = {"content": sample_letter_notification.template.content,
"subject": sample_letter_notification.template.subject}
letter = LetterDVLATemplate(t, sample_letter_notification.personalisation, "random-string")
assert str(letter) == "140|500|001||random-string|||||||||||||A1||A2|A3|A4|A5|A6|A_POST|||||||||23 March 2017<cr><cr><h1>Template subject<normal><cr><cr>Dear Sir/Madam, Hello. Yours Truly, The Government.<cr><cr>" # noqa
def test_send_inbound_sms_to_service_post_https_request_to_service(notify_api, sample_service):
inbound_api = create_service_inbound_api(service=sample_service, url="https://some.service.gov.uk/",
bearer_token="something_unique")
tests/app/dao/test_daily_sorted_letter_dao.py
+47
View File
@@ -0,0 +1,47 @@
from datetime import date
from app.dao.daily_sorted_letter_dao import (
dao_create_or_update_daily_sorted_letter,
dao_get_daily_sorted_letter_by_billing_day,
)
from app.models import DailySortedLetter
from tests.app.db import create_daily_sorted_letter
def test_dao_get_daily_sorted_letter_by_billing_day(notify_db, notify_db_session):
billing_day = date(2018, 2, 1)
other_day = date(2017, 9, 8)
daily_sorted_letters = create_daily_sorted_letter(billing_day=billing_day)
assert dao_get_daily_sorted_letter_by_billing_day(billing_day) == daily_sorted_letters
assert not dao_get_daily_sorted_letter_by_billing_day(other_day)
def test_dao_create_or_update_daily_sorted_letter_creates_a_new_entry(notify_db, notify_db_session):
billing_day = date(2018, 2, 1)
dsl = DailySortedLetter(billing_day=billing_day, unsorted_count=2, sorted_count=0)
dao_create_or_update_daily_sorted_letter(dsl)
daily_sorted_letter = dao_get_daily_sorted_letter_by_billing_day(billing_day)
assert daily_sorted_letter.billing_day == billing_day
assert daily_sorted_letter.unsorted_count == 2
assert daily_sorted_letter.sorted_count == 0
assert not daily_sorted_letter.updated_at
def test_dao_create_or_update_daily_sorted_letter_updates_an_existing_entry(
notify_db,
notify_db_session
):
create_daily_sorted_letter(unsorted_count=2, sorted_count=3)
dsl = DailySortedLetter(billing_day=date(2018, 1, 18), unsorted_count=5, sorted_count=17)
dao_create_or_update_daily_sorted_letter(dsl)
daily_sorted_letter = dao_get_daily_sorted_letter_by_billing_day(dsl.billing_day)
assert daily_sorted_letter.unsorted_count == 7
assert daily_sorted_letter.sorted_count == 20
assert daily_sorted_letter.updated_at
tests/app/dao/test_stats_template_usage_by_month_dao.py
+31 -1
View File
@@ -3,7 +3,7 @@ from app.dao.stats_template_usage_by_month_dao import (
insert_or_update_stats_for_template,
dao_get_template_usage_stats_by_service
)
from app.models import StatsTemplateUsageByMonth
from app.models import StatsTemplateUsageByMonth, LETTER_TYPE, PRECOMPILED_TEMPLATE_NAME
from tests.app.db import create_service, create_template
@@ -74,6 +74,36 @@ def test_dao_get_template_usage_stats_by_service(sample_service):
assert len(result) == 1
def test_dao_get_template_usage_stats_by_service_for_precompiled_letters(sample_service):
letter_template = create_template(service=sample_service, template_type=LETTER_TYPE)
precompiled_letter_template = create_template(
service=sample_service, template_name=PRECOMPILED_TEMPLATE_NAME, hidden=True, template_type=LETTER_TYPE)
db.session.add(StatsTemplateUsageByMonth(
template_id=letter_template.id,
month=5,
year=2017,
count=10
))
db.session.add(StatsTemplateUsageByMonth(
template_id=precompiled_letter_template.id,
month=4,
year=2017,
count=20
))
result = dao_get_template_usage_stats_by_service(sample_service.id, 2017)
assert len(result) == 2
assert [
(letter_template.id, 'letter Template Name', 'letter', False, 5, 2017, 10),
(precompiled_letter_template.id, PRECOMPILED_TEMPLATE_NAME, 'letter', True, 4, 2017, 20)
] == result
def test_dao_get_template_usage_stats_by_service_specific_year(sample_service):
email_template = create_template(service=sample_service, template_type="email")
tests/app/dao/test_templates_dao.py
+34 -4
View File
@@ -13,7 +13,12 @@ from app.dao.templates_dao import (
dao_get_templates_for_cache,
dao_redact_template, dao_update_template_reply_to
)
from app.models import Template, TemplateHistory, TemplateRedacted
from app.models import (
Template,
TemplateHistory,
TemplateRedacted,
PRECOMPILED_TEMPLATE_NAME
)
from tests.app.conftest import sample_template as create_sample_template
from tests.app.db import create_template, create_letter_contact
@@ -503,8 +508,33 @@ def test_get_templates_by_ids_successful(notify_db, notify_db_session):
cache = [[k, v] for k, v in sample_cache_dict.items()]
templates = dao_get_templates_for_cache(cache)
assert len(templates) == 2
assert [(template_1.id, template_1.template_type, template_1.name, 2),
(template_2.id, template_2.template_type, template_2.name, 3)] == templates
assert [(template_1.id, template_1.template_type, template_1.name, False, 2),
(template_2.id, template_2.template_type, template_2.name, False, 3)] == templates
def test_get_letter_templates_by_ids_successful(notify_db, notify_db_session):
template_1 = create_sample_template(
notify_db,
notify_db_session,
template_name=PRECOMPILED_TEMPLATE_NAME,
template_type="letter",
content="Template content",
hidden=True
)
template_2 = create_sample_template(
notify_db,
notify_db_session,
template_name='Sample Template 2',
template_type="letter",
content="Template content"
)
sample_cache_dict = {str.encode(str(template_1.id)): str.encode('2'),
str.encode(str(template_2.id)): str.encode('3')}
cache = [[k, v] for k, v in sample_cache_dict.items()]
templates = dao_get_templates_for_cache(cache)
assert len(templates) == 2
assert [(template_1.id, template_1.template_type, template_1.name, True, 2),
(template_2.id, template_2.template_type, template_2.name, False, 3)] == templates
def test_get_templates_by_ids_successful_for_one_cache_item(notify_db, notify_db_session):
@@ -519,7 +549,7 @@ def test_get_templates_by_ids_successful_for_one_cache_item(notify_db, notify_db
cache = [[k, v] for k, v in sample_cache_dict.items()]
templates = dao_get_templates_for_cache(cache)
assert len(templates) == 1
assert [(template_1.id, template_1.template_type, template_1.name, 2)] == templates
assert [(template_1.id, template_1.template_type, template_1.name, False, 2)] == templates
def test_get_templates_by_ids_returns_empty_list():
tests/app/db.py
+18 -2
View File
@@ -1,4 +1,4 @@
from datetime import datetime
from datetime import datetime, date
import uuid
from app import db
@@ -9,6 +9,7 @@ from app.dao.service_sms_sender_dao import update_existing_sms_sender_with_inbou
from app.dao.invited_org_user_dao import save_invited_org_user
from app.models import (
ApiKey,
DailySortedLetter,
InboundSms,
InboundNumber,
Job,
@@ -130,7 +131,8 @@ def create_template(
template_name=None,
subject='Template subject',
content='Dear Sir/Madam, Hello. Yours Truly, The Government.',
reply_to=None
reply_to=None,
hidden=False
):
data = {
'name': template_name or '{} Template Name'.format(template_type),
@@ -139,6 +141,7 @@ def create_template(
'service': service,
'created_by': service.created_by,
'reply_to': reply_to,
'hidden': hidden
}
if template_type != SMS_TYPE:
data['subject'] = subject
@@ -504,3 +507,16 @@ def create_invited_org_user(organisation, invited_by, email_address='invite@exam
)
save_invited_org_user(invited_org_user)
return invited_org_user
def create_daily_sorted_letter(billing_day=date(2018, 1, 18), unsorted_count=0, sorted_count=0):
daily_sorted_letter = DailySortedLetter(
billing_day=billing_day,
unsorted_count=unsorted_count,
sorted_count=sorted_count
)
db.session.add(daily_sorted_letter)
db.session.commit()
return daily_sorted_letter
tests/app/delivery/test_send_to_providers.py
+1 -41
View File
@@ -1,7 +1,7 @@
import uuid
from collections import namedtuple
from datetime import datetime
from unittest.mock import ANY, call
from unittest.mock import ANY
import pytest
from flask import current_app
@@ -75,7 +75,6 @@ def test_should_send_personalised_template_to_correct_sms_provider_and_persist(
reply_to_text=sample_sms_template_with_html.service.get_default_sms_sender())
mocker.patch('app.mmg_client.send_sms')
stats_mock = mocker.patch('app.delivery.send_to_providers.create_initial_notification_statistic_tasks')
send_to_providers.send_sms_to_provider(
db_notification
@@ -88,8 +87,6 @@ def test_should_send_personalised_template_to_correct_sms_provider_and_persist(
sender=current_app.config['FROM_NUMBER']
)
stats_mock.assert_called_once_with(db_notification)
notification = Notification.query.filter_by(id=db_notification.id).one()
assert notification.status == 'sending'
@@ -110,7 +107,6 @@ def test_should_send_personalised_template_to_correct_email_provider_and_persist
)
mocker.patch('app.aws_ses_client.send_email', return_value='reference')
stats_mock = mocker.patch('app.delivery.send_to_providers.create_initial_notification_statistic_tasks')
send_to_providers.send_email_to_provider(
db_notification
@@ -124,7 +120,6 @@ def test_should_send_personalised_template_to_correct_email_provider_and_persist
html_body=ANY,
reply_to_address=None
)
stats_mock.assert_called_once_with(db_notification)
assert '<!DOCTYPE html' in app.aws_ses_client.send_email.call_args[1]['html_body']
assert '&lt;em&gt;some HTML&lt;/em&gt;' in app.aws_ses_client.send_email.call_args[1]['html_body']
@@ -141,11 +136,9 @@ def test_should_not_send_email_message_when_service_is_inactive_notifcation_is_i
):
sample_service.active = False
send_mock = mocker.patch("app.aws_ses_client.send_email", return_value='reference')
stats_mock = mocker.patch('app.delivery.send_to_providers.create_initial_notification_statistic_tasks')
send_to_providers.send_email_to_provider(sample_notification)
send_mock.assert_not_called()
stats_mock.assert_not_called()
assert Notification.query.get(sample_notification.id).status == 'technical-failure'
@@ -154,11 +147,9 @@ def test_should_not_send_sms_message_when_service_is_inactive_notifcation_is_in_
sample_service, sample_notification, mocker, client_send):
sample_service.active = False
send_mock = mocker.patch(client_send, return_value='reference')
stats_mock = mocker.patch('app.delivery.send_to_providers.create_initial_notification_statistic_tasks')
send_to_providers.send_sms_to_provider(sample_notification)
send_mock.assert_not_called()
stats_mock.assert_not_called()
assert Notification.query.get(sample_notification.id).status == 'technical-failure'
@@ -169,7 +160,6 @@ def test_send_sms_should_use_template_version_from_notification_not_latest(
reply_to_text=sample_template.service.get_default_sms_sender())
mocker.patch('app.mmg_client.send_sms')
mocker.patch('app.delivery.send_to_providers.create_initial_notification_statistic_tasks')
version_on_notification = sample_template.version
@@ -209,7 +199,6 @@ def test_should_call_send_sms_response_task_if_research_mode(
):
mocker.patch('app.mmg_client.send_sms')
mocker.patch('app.delivery.send_to_providers.send_sms_response')
stats_mock = mocker.patch('app.delivery.send_to_providers.create_initial_notification_statistic_tasks')
if research_mode:
sample_service.research_mode = True
@@ -222,7 +211,6 @@ def test_should_call_send_sms_response_task_if_research_mode(
sample_notification
)
assert not mmg_client.send_sms.called
stats_mock.assert_called_once_with(sample_notification)
app.delivery.send_to_providers.send_sms_response.assert_called_once_with(
'mmg', str(sample_notification.id), sample_notification.to
@@ -260,7 +248,6 @@ def test_should_set_billable_units_to_zero_in_research_mode_or_test_key(
mocker.patch('app.mmg_client.send_sms')
mocker.patch('app.delivery.send_to_providers.send_sms_response')
stats_mock = mocker.patch('app.delivery.send_to_providers.create_initial_notification_statistic_tasks')
if research_mode:
sample_service.research_mode = True
@@ -271,7 +258,6 @@ def test_should_set_billable_units_to_zero_in_research_mode_or_test_key(
send_to_providers.send_sms_to_provider(
sample_notification
)
stats_mock.assert_called_once_with(sample_notification)
assert notifications_dao.get_notification_by_id(sample_notification.id).billable_units == 0
@@ -282,7 +268,6 @@ def test_should_not_send_to_provider_when_status_is_not_created(
notification = create_notification(template=sample_template, status='sending')
mocker.patch('app.mmg_client.send_sms')
response_mock = mocker.patch('app.delivery.send_to_providers.send_sms_response')
stats_mock = mocker.patch('app.delivery.send_to_providers.create_initial_notification_statistic_tasks')
send_to_providers.send_sms_to_provider(
notification
@@ -290,7 +275,6 @@ def test_should_not_send_to_provider_when_status_is_not_created(
app.mmg_client.send_sms.assert_not_called()
response_mock.assert_not_called()
stats_mock.assert_not_called()
def test_should_send_sms_with_downgraded_content(notify_db_session, mocker):
@@ -307,7 +291,6 @@ def test_should_send_sms_with_downgraded_content(notify_db_session, mocker):
)
mocker.patch('app.mmg_client.send_sms')
mocker.patch('app.delivery.send_to_providers.create_initial_notification_statistic_tasks')
send_to_providers.send_sms_to_provider(db_notification)
@@ -324,7 +307,6 @@ def test_send_sms_should_use_service_sms_sender(
sample_template,
mocker):
mocker.patch('app.mmg_client.send_sms')
mocker.patch('app.delivery.send_to_providers.create_initial_notification_statistic_tasks')
sms_sender = create_service_sms_sender(service=sample_service, sms_sender='123456', is_default=False)
db_notification = create_notification(template=sample_template, sms_sender_id=sms_sender.id,
@@ -363,14 +345,12 @@ def test_send_email_to_provider_should_call_research_mode_task_response_task_if_
mocker.patch('app.uuid.uuid4', return_value=reference)
mocker.patch('app.aws_ses_client.send_email')
mocker.patch('app.delivery.send_to_providers.send_email_response')
stats_mock = mocker.patch('app.delivery.send_to_providers.create_initial_notification_statistic_tasks')
send_to_providers.send_email_to_provider(
notification
)
assert not app.aws_ses_client.send_email.called
stats_mock.assert_called_once_with(notification)
app.delivery.send_to_providers.send_email_response.assert_called_once_with(str(reference), 'john@smith.com')
persisted_notification = Notification.query.filter_by(id=notification.id).one()
assert persisted_notification.to == 'john@smith.com'
@@ -390,12 +370,10 @@ def test_send_email_to_provider_should_not_send_to_provider_when_status_is_not_c
notification = create_notification(template=sample_email_template, status='sending')
mocker.patch('app.aws_ses_client.send_email')
mocker.patch('app.delivery.send_to_providers.send_email_response')
stats_mock = mocker.patch('app.delivery.send_to_providers.create_initial_notification_statistic_tasks')
send_to_providers.send_sms_to_provider(
notification
)
stats_mock.assert_not_called()
app.aws_ses_client.send_email.assert_not_called()
app.delivery.send_to_providers.send_email_response.assert_not_called()
@@ -405,7 +383,6 @@ def test_send_email_should_use_service_reply_to_email(
sample_email_template,
mocker):
mocker.patch('app.aws_ses_client.send_email', return_value='reference')
mocker.patch('app.delivery.send_to_providers.create_initial_notification_statistic_tasks')
db_notification = create_notification(template=sample_email_template, reply_to_text='foo@bar.com')
create_reply_to_email(service=sample_service, email_address='foo@bar.com')
@@ -512,7 +489,6 @@ def test_get_logo_url_works_for_different_environments(base_url, expected_url):
def test_should_not_set_billable_units_if_research_mode(notify_db, sample_service, sample_notification, mocker):
mocker.patch('app.mmg_client.send_sms')
mocker.patch('app.delivery.send_to_providers.send_sms_response')
stats_mock = mocker.patch('app.delivery.send_to_providers.create_initial_notification_statistic_tasks')
sample_service.research_mode = True
notify_db.session.add(sample_service)
@@ -521,7 +497,6 @@ def test_should_not_set_billable_units_if_research_mode(notify_db, sample_servic
send_to_providers.send_sms_to_provider(
sample_notification
)
stats_mock.assert_called_once_with(sample_notification)
persisted_notification = notifications_dao.get_notification_by_id(sample_notification.id)
assert persisted_notification.billable_units == 0
@@ -545,7 +520,6 @@ def test_should_update_billable_units_according_to_research_mode_and_key_type(
):
mocker.patch('app.mmg_client.send_sms')
mocker.patch('app.delivery.send_to_providers.send_sms_response')
stats_mock = mocker.patch('app.delivery.send_to_providers.create_initial_notification_statistic_tasks')
if research_mode:
sample_service.research_mode = True
@@ -557,7 +531,6 @@ def test_should_update_billable_units_according_to_research_mode_and_key_type(
send_to_providers.send_sms_to_provider(
sample_notification
)
stats_mock.assert_called_once_with(sample_notification)
assert sample_notification.billable_units == billable_units
@@ -591,7 +564,6 @@ def test_should_send_sms_to_international_providers(
mocker.patch('app.mmg_client.send_sms')
mocker.patch('app.firetext_client.send_sms')
stats_mock = mocker.patch('app.delivery.send_to_providers.create_initial_notification_statistic_tasks')
send_to_providers.send_sms_to_provider(
db_notification_uk
@@ -618,11 +590,6 @@ def test_should_send_sms_to_international_providers(
notification_uk = Notification.query.filter_by(id=db_notification_uk.id).one()
notification_int = Notification.query.filter_by(id=db_notification_international.id).one()
stats_mock.assert_has_calls([
call(db_notification_uk),
call(db_notification_international)
])
assert notification_uk.status == 'sending'
assert notification_uk.sent_by == 'firetext'
assert notification_int.status == 'sent'
@@ -642,7 +609,6 @@ def test_should_send_international_sms_with_formatted_phone_number(
send_notification_mock = mocker.patch('app.mmg_client.send_sms')
mocker.patch('app.delivery.send_to_providers.send_sms_response')
mocker.patch('app.delivery.send_to_providers.create_initial_notification_statistic_tasks')
send_to_providers.send_sms_to_provider(
notification
@@ -664,7 +630,6 @@ def test_should_set_international_phone_number_to_sent_status(
mocker.patch('app.mmg_client.send_sms')
mocker.patch('app.delivery.send_to_providers.send_sms_response')
mocker.patch('app.delivery.send_to_providers.create_initial_notification_statistic_tasks')
send_to_providers.send_sms_to_provider(
notification
@@ -691,7 +656,6 @@ def test_should_handle_sms_sender_and_prefix_message(
notify_db_session
):
mocker.patch('app.mmg_client.send_sms')
mocker.patch('app.delivery.send_to_providers.create_initial_notification_statistic_tasks')
service = create_service_with_defined_sms_sender(sms_sender_value=sms_sender, prefix_sms=prefix_sms)
template = create_template(service, content='bar')
notification = create_notification(template, reply_to_text=sms_sender)
@@ -710,7 +674,6 @@ def test_send_email_to_provider_uses_reply_to_from_notification(
sample_email_template,
mocker):
mocker.patch('app.aws_ses_client.send_email', return_value='reference')
mocker.patch('app.delivery.send_to_providers.create_initial_notification_statistic_tasks')
db_notification = create_notification(template=sample_email_template, reply_to_text="test@test.com")
@@ -732,7 +695,6 @@ def test_send_email_to_provider_should_format_reply_to_email_address(
sample_email_template,
mocker):
mocker.patch('app.aws_ses_client.send_email', return_value='reference')
mocker.patch('app.delivery.send_to_providers.create_initial_notification_statistic_tasks')
db_notification = create_notification(template=sample_email_template, reply_to_text="test@test.com\t")
@@ -753,7 +715,6 @@ def test_send_email_to_provider_should_format_reply_to_email_address(
def test_send_sms_to_provider_should_format_phone_number(sample_notification, mocker):
sample_notification.to = '+44 (7123) 123-123'
send_mock = mocker.patch('app.mmg_client.send_sms')
mocker.patch('app.delivery.send_to_providers.create_initial_notification_statistic_tasks')
send_to_providers.send_sms_to_provider(sample_notification)
@@ -763,7 +724,6 @@ def test_send_sms_to_provider_should_format_phone_number(sample_notification, mo
def test_send_email_to_provider_should_format_email_address(sample_email_notification, mocker):
sample_email_notification.to = 'test@example.com\t'
send_mock = mocker.patch('app.aws_ses_client.send_email', return_value='reference')
mocker.patch('app.delivery.send_to_providers.create_initial_notification_statistic_tasks')
send_to_providers.send_email_to_provider(sample_email_notification)
tests/app/letters/test_letter_utils.py
+18
View File
@@ -0,0 +1,18 @@
import pytest
from app.letters.utils import get_bucket_prefix_for_notification
def test_get_bucket_prefix_for_notification_valid_notification(sample_notification):
bucket_prefix = get_bucket_prefix_for_notification(sample_notification)
assert bucket_prefix == '{folder}/NOTIFY.{reference}'.format(
folder=sample_notification.created_at.date(),
reference=sample_notification.reference
).upper()
def test_get_bucket_prefix_for_notification_invalid_notification():
with pytest.raises(AttributeError):
get_bucket_prefix_for_notification(None)
tests/app/notifications/test_notifications_ses_callback.py
+3 -31
View File
@@ -1,5 +1,4 @@
from datetime import datetime
from unittest.mock import call
from flask import json
from freezegun import freeze_time
@@ -22,9 +21,6 @@ def test_ses_callback_should_update_notification_status(
with freeze_time('2001-01-01T12:00:00'):
mocker.patch('app.statsd_client.incr')
mocker.patch('app.statsd_client.timing_with_dates')
stats_mock = mocker.patch(
'app.notifications.notifications_ses_callback.create_outcome_notification_statistic_tasks'
)
send_mock = mocker.patch(
'app.celery.service_callback_tasks.send_delivery_status_to_service.apply_async'
)
@@ -46,7 +42,6 @@ def test_ses_callback_should_update_notification_status(
"callback.ses.elapsed-time", datetime.utcnow(), notification.sent_at
)
statsd_client.incr.assert_any_call("callback.ses.delivered")
stats_mock.assert_called_once_with(notification)
send_mock.assert_called_once_with([str(notification.id)], queue="service-callbacks")
@@ -86,13 +81,10 @@ def test_ses_callback_should_update_multiple_notification_status_sent(
sample_email_template,
mocker):
stats_mock = mocker.patch(
'app.notifications.notifications_ses_callback.create_outcome_notification_statistic_tasks'
)
send_mock = mocker.patch(
'app.celery.service_callback_tasks.send_delivery_status_to_service.apply_async'
)
notification1 = create_sample_notification(
create_sample_notification(
notify_db,
notify_db_session,
template=sample_email_template,
@@ -100,7 +92,7 @@ def test_ses_callback_should_update_multiple_notification_status_sent(
sent_at=datetime.utcnow(),
status='sending')
notification2 = create_sample_notification(
create_sample_notification(
notify_db,
notify_db_session,
template=sample_email_template,
@@ -108,7 +100,7 @@ def test_ses_callback_should_update_multiple_notification_status_sent(
sent_at=datetime.utcnow(),
status='sending')
notification3 = create_sample_notification(
create_sample_notification(
notify_db,
notify_db_session,
template=sample_email_template,
@@ -119,12 +111,6 @@ def test_ses_callback_should_update_multiple_notification_status_sent(
assert process_ses_response(ses_notification_callback(reference='ref1')) is None
assert process_ses_response(ses_notification_callback(reference='ref2')) is None
assert process_ses_response(ses_notification_callback(reference='ref3')) is None
stats_mock.assert_has_calls([
call(notification1),
call(notification2),
call(notification3)
])
assert send_mock.called
@@ -133,10 +119,6 @@ def test_ses_callback_should_set_status_to_temporary_failure(client,
notify_db_session,
sample_email_template,
mocker):
stats_mock = mocker.patch(
'app.notifications.notifications_ses_callback.create_outcome_notification_statistic_tasks'
)
send_mock = mocker.patch(
'app.celery.service_callback_tasks.send_delivery_status_to_service.apply_async'
)
@@ -153,7 +135,6 @@ def test_ses_callback_should_set_status_to_temporary_failure(client,
assert process_ses_response(ses_soft_bounce_callback(reference='ref')) is None
assert get_notification_by_id(notification.id).status == 'temporary-failure'
assert send_mock.called
stats_mock.assert_called_once_with(notification)
def test_ses_callback_should_not_set_status_once_status_is_delivered(client,
@@ -161,10 +142,6 @@ def test_ses_callback_should_not_set_status_once_status_is_delivered(client,
notify_db_session,
sample_email_template,
mocker):
stats_mock = mocker.patch(
'app.notifications.notifications_ses_callback.create_outcome_notification_statistic_tasks'
)
notification = create_sample_notification(
notify_db,
notify_db_session,
@@ -175,7 +152,6 @@ def test_ses_callback_should_not_set_status_once_status_is_delivered(client,
)
assert get_notification_by_id(notification.id).status == 'delivered'
stats_mock.assert_not_called()
def test_ses_callback_should_set_status_to_permanent_failure(client,
@@ -183,9 +159,6 @@ def test_ses_callback_should_set_status_to_permanent_failure(client,
notify_db_session,
sample_email_template,
mocker):
stats_mock = mocker.patch(
'app.notifications.notifications_ses_callback.create_outcome_notification_statistic_tasks'
)
send_mock = mocker.patch(
'app.celery.service_callback_tasks.send_delivery_status_to_service.apply_async'
)
@@ -203,7 +176,6 @@ def test_ses_callback_should_set_status_to_permanent_failure(client,
assert process_ses_response(ses_hard_bounce_callback(reference='ref')) is None
assert get_notification_by_id(notification.id).status == 'permanent-failure'
assert send_mock.called
stats_mock.assert_called_once_with(notification)
def test_remove_emails_from_bounce():
tests/app/notifications/test_process_client_response.py
-10
View File
@@ -50,7 +50,6 @@ def test_validate_callback_data_returns_error_for_empty_string():
def test_outcome_statistics_called_for_successful_callback(sample_notification, mocker):
stats_mock = mocker.patch('app.notifications.process_client_response.create_outcome_notification_statistic_tasks')
mocker.patch(
'app.notifications.process_client_response.notifications_dao.update_notification_status_by_id',
return_value=sample_notification
@@ -65,7 +64,6 @@ def test_outcome_statistics_called_for_successful_callback(sample_notification,
assert success == "MMG callback succeeded. reference {} updated".format(str(reference))
assert error is None
send_mock.assert_called_once_with([str(sample_notification.id)], queue="service-callbacks")
stats_mock.assert_called_once_with(sample_notification)
def test_sms_resonse_does_not_call_send_callback_if_no_db_entry(sample_notification, mocker):
@@ -82,30 +80,22 @@ def test_sms_resonse_does_not_call_send_callback_if_no_db_entry(sample_notificat
def test_process_sms_response_return_success_for_send_sms_code_reference(mocker):
stats_mock = mocker.patch('app.notifications.process_client_response.create_outcome_notification_statistic_tasks')
success, error = process_sms_client_response(status='000', reference='send-sms-code', client_name='sms-client')
assert success == "{} callback succeeded: send-sms-code".format('sms-client')
assert error is None
stats_mock.assert_not_called()
def test_process_sms_response_returns_error_bad_reference(mocker):
stats_mock = mocker.patch('app.notifications.process_client_response.create_outcome_notification_statistic_tasks')
success, error = process_sms_client_response(status='000', reference='something-bad', client_name='sms-client')
assert success is None
assert error == "{} callback with invalid reference {}".format('sms-client', 'something-bad')
stats_mock.assert_not_called()
def test_process_sms_response_raises_client_exception_for_unknown_sms_client(mocker):
stats_mock = mocker.patch('app.notifications.process_client_response.create_outcome_notification_statistic_tasks')
success, error = process_sms_client_response(status='000', reference=str(uuid.uuid4()), client_name='sms-client')
assert success is None
assert error == 'unknown sms client: {}'.format('sms-client')
stats_mock.assert_not_called()
def test_process_sms_response_raises_client_exception_for_unknown_status(mocker):
tests/app/organisation/test_rest.py
+99
View File
@@ -1,5 +1,7 @@
import uuid
import pytest
from app.models import Organisation
from app.dao.organisation_dao import dao_add_service_to_organisation, dao_add_user_to_organisation
from tests.app.db import create_organisation, create_service, create_user
@@ -310,3 +312,100 @@ def test_get_organisation_users_returns_users_for_organisation(admin_request, sa
assert len(response['data']) == 2
assert response['data'][0]['id'] == str(first.id)
def test_is_organisation_name_unique_returns_200_if_unique(admin_request, notify_db, notify_db_session):
organisation = create_organisation(name='unique')
response = admin_request.get(
'organisation.is_organisation_name_unique',
_expected_status=200,
org_id=organisation.id,
name='something'
)
assert response == {"result": True}
@pytest.mark.parametrize('name', ["UNIQUE", "Unique.", "**uniQUE**"])
def test_is_organisation_name_unique_returns_200_and_name_capitalized_or_punctuation_added(
admin_request,
notify_db,
notify_db_session,
name
):
organisation = create_organisation(name='unique')
response = admin_request.get(
'organisation.is_organisation_name_unique',
_expected_status=200,
org_id=organisation.id,
name=name
)
assert response == {"result": True}
@pytest.mark.parametrize('name', ["UNIQUE", "Unique"])
def test_is_organisation_name_unique_returns_200_and_false_with_same_name_and_different_case_of_other_organisation(
admin_request,
notify_db,
notify_db_session,
name
):
create_organisation(name='unique')
different_organisation_id = '111aa111-2222-bbbb-aaaa-111111111111'
response = admin_request.get(
'organisation.is_organisation_name_unique',
_expected_status=200,
org_id=different_organisation_id,
name=name
)
assert response == {"result": False}
def test_is_organisation_name_unique_returns_200_and_false_if_name_exists_for_a_different_organisation(
admin_request,
notify_db,
notify_db_session
):
create_organisation(name='existing name')
different_organisation_id = '111aa111-2222-bbbb-aaaa-111111111111'
response = admin_request.get(
'organisation.is_organisation_name_unique',
_expected_status=200,
org_id=different_organisation_id,
name='existing name'
)
assert response == {"result": False}
def test_is_organisation_name_unique_returns_200_and_true_if_name_exists_for_the_same_organisation(
admin_request,
notify_db,
notify_db_session
):
organisation = create_organisation(name='unique')
response = admin_request.get(
'organisation.is_organisation_name_unique',
_expected_status=200,
org_id=organisation.id,
name='unique'
)
assert response == {"result": True}
def test_is_organisation_name_unique_returns_400_when_name_does_not_exist(admin_request):
response = admin_request.get(
'organisation.is_organisation_name_unique',
_expected_status=400
)
assert response["message"][0]["org_id"] == ["Can't be empty"]
assert response["message"][1]["name"] == ["Can't be empty"]
tests/app/performance_platform/test_total_sent_notifications.py
+16 -2
View File
@@ -6,7 +6,7 @@ from freezegun import freeze_time
from app.utils import get_midnight_for_day_before
from app.performance_platform.total_sent_notifications import (
send_total_notifications_sent_for_day_stats,
get_total_sent_notifications_yesterday
get_total_sent_notifications_for_day
)
from tests.app.conftest import (
@@ -55,14 +55,16 @@ def test_get_total_sent_notifications_yesterday_returns_expected_totals_dict(
# Create some notifications for the day before
yesterday = datetime(2016, 1, 10, 15, 30, 0, 0)
ereyesterday = datetime(2016, 1, 9, 15, 30, 0, 0)
with freeze_time(yesterday):
notification_history(notification_type='letter')
notification_history(notification_type='sms')
notification_history(notification_type='sms')
notification_history(notification_type='email')
notification_history(notification_type='email')
notification_history(notification_type='email')
total_count_dict = get_total_sent_notifications_yesterday()
total_count_dict = get_total_sent_notifications_for_day(yesterday)
assert total_count_dict == {
"start_date": get_midnight_for_day_before(datetime.utcnow()),
@@ -71,5 +73,17 @@ def test_get_total_sent_notifications_yesterday_returns_expected_totals_dict(
},
"sms": {
"count": 2
},
"letter": {
"count": 1
}
}
another_day = get_total_sent_notifications_for_day(ereyesterday)
assert another_day == {
'email': {'count': 0},
'letter': {'count': 0},
'sms': {'count': 0},
'start_date': datetime(2016, 1, 9, 0, 0),
}
tests/app/service/test_rest.py
+49 -2
View File
@@ -25,7 +25,8 @@ from app.models import (
User,
DVLA_ORG_LAND_REGISTRY,
KEY_TYPE_NORMAL, KEY_TYPE_TEAM, KEY_TYPE_TEST,
EMAIL_TYPE, SMS_TYPE, LETTER_TYPE, INTERNATIONAL_SMS_TYPE, INBOUND_SMS_TYPE
EMAIL_TYPE, SMS_TYPE, LETTER_TYPE, INTERNATIONAL_SMS_TYPE, INBOUND_SMS_TYPE,
PRECOMPILED_TEMPLATE_NAME
)
from tests import create_authorization_header
from tests.app.conftest import (
@@ -1831,7 +1832,12 @@ def test_get_template_usage_by_month_returns_two_templates(
sample_service
):
template_one = create_template(sample_service)
template_one = create_template(
sample_service,
template_type=LETTER_TYPE,
template_name=PRECOMPILED_TEMPLATE_NAME,
hidden=True
)
# add a historical notification for template
not1 = create_notification_history(
@@ -1889,6 +1895,7 @@ def test_get_template_usage_by_month_returns_two_templates(
assert resp_json[0]["month"] == 4
assert resp_json[0]["year"] == 2017
assert resp_json[0]["count"] == 1
assert resp_json[0]["is_precompiled_letter"] is True
assert resp_json[1]["template_id"] == str(sample_template.id)
assert resp_json[1]["name"] == sample_template.name
@@ -1896,6 +1903,7 @@ def test_get_template_usage_by_month_returns_two_templates(
assert resp_json[1]["month"] == 4
assert resp_json[1]["year"] == 2017
assert resp_json[1]["count"] == 3
assert resp_json[1]["is_precompiled_letter"] is False
assert resp_json[2]["template_id"] == str(sample_template.id)
assert resp_json[2]["name"] == sample_template.name
@@ -1903,6 +1911,7 @@ def test_get_template_usage_by_month_returns_two_templates(
assert resp_json[2]["month"] == 11
assert resp_json[2]["year"] == 2017
assert resp_json[2]["count"] == 1
assert resp_json[2]["is_precompiled_letter"] is False
def test_search_for_notification_by_to_field(client, notify_db, notify_db_session):
@@ -2139,6 +2148,17 @@ def test_get_notification_for_service_includes_template_redacted(admin_request,
assert resp['template']['redact_personalisation'] is False
def test_get_notification_for_service_includes_precompiled_letter(admin_request, sample_notification):
resp = admin_request.get(
'service.get_notification_for_service',
service_id=sample_notification.service_id,
notification_id=sample_notification.id
)
assert resp['id'] == str(sample_notification.id)
assert resp['template']['is_precompiled_letter'] is False
def test_get_all_notifications_for_service_includes_template_redacted(admin_request, sample_service):
normal_template = create_template(sample_service)
@@ -2162,6 +2182,33 @@ def test_get_all_notifications_for_service_includes_template_redacted(admin_requ
assert resp['notifications'][1]['template']['redact_personalisation'] is True
def test_get_all_notifications_for_service_includes_template_hidden(admin_request, sample_service):
letter_template = create_template(sample_service, template_type=LETTER_TYPE)
precompiled_template = create_template(
sample_service,
template_type=LETTER_TYPE,
template_name='Pre-compiled PDF',
subject='Pre-compiled PDF',
hidden=True
)
with freeze_time('2000-01-01'):
letter_noti = create_notification(letter_template)
with freeze_time('2000-01-02'):
precompiled_noti = create_notification(precompiled_template)
resp = admin_request.get(
'service.get_all_notifications_for_service',
service_id=sample_service.id
)
assert resp['notifications'][0]['id'] == str(precompiled_noti.id)
assert resp['notifications'][0]['template']['is_precompiled_letter'] is True
assert resp['notifications'][1]['id'] == str(letter_noti.id)
assert resp['notifications'][1]['template']['is_precompiled_letter'] is False
def test_search_for_notification_by_to_field_returns_personlisation(
client,
notify_db,
tests/app/template/test_rest.py
+223 -5
View File
@@ -4,7 +4,9 @@ import random
import string
from datetime import datetime, timedelta
import botocore
import pytest
import requests_mock
from freezegun import freeze_time
from app.models import Template, SMS_TYPE, EMAIL_TYPE, LETTER_TYPE, TemplateHistory
@@ -16,7 +18,7 @@ from tests.app.conftest import (
sample_template_without_email_permission,
sample_template_without_letter_permission,
sample_template_without_sms_permission)
from tests.app.db import create_service, create_letter_contact, create_template
from tests.app.db import create_service, create_letter_contact, create_template, create_notification
from tests.conftest import set_config_values
@@ -823,11 +825,10 @@ def test_preview_letter_template_by_id_valid_file_type(
'TEMPLATE_PREVIEW_API_HOST': 'http://localhost/notifications-template-preview',
'TEMPLATE_PREVIEW_API_KEY': 'test-key'
}):
import requests_mock
with requests_mock.Mocker() as request_mock:
content = b'\x00\x01'
request_mock.post(
mock_post = request_mock.post(
'http://localhost/notifications-template-preview/preview.pdf',
content=content,
headers={'X-pdf-page-count': '1'},
@@ -841,6 +842,7 @@ def test_preview_letter_template_by_id_valid_file_type(
file_type='pdf'
)
assert mock_post.last_request.json()
assert base64.b64decode(resp['content']) == content
@@ -858,7 +860,7 @@ def test_preview_letter_template_by_id_template_preview_500(
with requests_mock.Mocker() as request_mock:
content = b'\x00\x01'
request_mock.post(
mock_post = request_mock.post(
'http://localhost/notifications-template-preview/preview.pdf',
content=content,
headers={'X-pdf-page-count': '1'},
@@ -873,4 +875,220 @@ def test_preview_letter_template_by_id_template_preview_500(
_expected_status=500
)
assert resp['message'] == 'Error generating preview for {}'.format(sample_letter_notification.id)
assert mock_post.last_request.json()
assert 'Status code: 404' in resp['message']
assert 'Error generating preview letter for {}'.format(sample_letter_notification.id) in resp['message']
def test_preview_letter_template_precompiled_pdf_file_type(
notify_api,
client,
admin_request,
sample_service,
mocker
):
template = create_template(sample_service,
template_type='letter',
template_name='Pre-compiled PDF',
subject='Pre-compiled PDF',
hidden=True)
notification = create_notification(template)
with set_config_values(notify_api, {
'TEMPLATE_PREVIEW_API_HOST': 'http://localhost/notifications-template-preview',
'TEMPLATE_PREVIEW_API_KEY': 'test-key'
}):
with requests_mock.Mocker():
content = b'\x00\x01'
mock_get_letter_pdf = mocker.patch('app.template.rest.get_letter_pdf', return_value=content)
resp = admin_request.get(
'template.preview_letter_template_by_notification_id',
service_id=notification.service_id,
notification_id=notification.id,
file_type='pdf'
)
assert mock_get_letter_pdf.called_once_with(notification)
assert base64.b64decode(resp['content']) == content
def test_preview_letter_template_precompiled_s3_error(
notify_api,
client,
admin_request,
sample_service,
mocker
):
template = create_template(sample_service,
template_type='letter',
template_name='Pre-compiled PDF',
subject='Pre-compiled PDF',
hidden=True)
notification = create_notification(template)
with set_config_values(notify_api, {
'TEMPLATE_PREVIEW_API_HOST': 'http://localhost/notifications-template-preview',
'TEMPLATE_PREVIEW_API_KEY': 'test-key'
}):
with requests_mock.Mocker():
mocker.patch('app.template.rest.get_letter_pdf',
side_effect=botocore.exceptions.ClientError(
{'Error': {'Code': '403', 'Message': 'Unauthorized'}},
'GetObject'
))
admin_request.get(
'template.preview_letter_template_by_notification_id',
service_id=notification.service_id,
notification_id=notification.id,
file_type='pdf',
_expected_status=500
)
def test_preview_letter_template_precompiled_png_file_type(
notify_api,
client,
admin_request,
sample_service,
mocker
):
template = create_template(sample_service,
template_type='letter',
template_name='Pre-compiled PDF',
subject='Pre-compiled PDF',
hidden=True)
notification = create_notification(template)
with set_config_values(notify_api, {
'TEMPLATE_PREVIEW_API_HOST': 'http://localhost/notifications-template-preview',
'TEMPLATE_PREVIEW_API_KEY': 'test-key'
}):
with requests_mock.Mocker() as request_mock:
pdf_content = b'\x00\x01'
png_content = b'\x00\x02'
mock_get_letter_pdf = mocker.patch('app.template.rest.get_letter_pdf', return_value=pdf_content)
mock_post = request_mock.post(
'http://localhost/notifications-template-preview/precompiled-preview.png',
content=png_content,
headers={'X-pdf-page-count': '1'},
status_code=200
)
resp = admin_request.get(
'template.preview_letter_template_by_notification_id',
service_id=notification.service_id,
notification_id=notification.id,
file_type='png'
)
with pytest.raises(ValueError):
mock_post.last_request.json()
assert mock_get_letter_pdf.called_once_with(notification)
assert base64.b64decode(resp['content']) == png_content
def test_preview_letter_template_precompiled_png_template_preview_500_error(
notify_api,
client,
admin_request,
sample_service,
mocker
):
template = create_template(sample_service,
template_type='letter',
template_name='Pre-compiled PDF',
subject='Pre-compiled PDF',
hidden=True)
notification = create_notification(template)
with set_config_values(notify_api, {
'TEMPLATE_PREVIEW_API_HOST': 'http://localhost/notifications-template-preview',
'TEMPLATE_PREVIEW_API_KEY': 'test-key'
}):
with requests_mock.Mocker() as request_mock:
pdf_content = b'\x00\x01'
png_content = b'\x00\x02'
mocker.patch('app.template.rest.get_letter_pdf', return_value=pdf_content)
mock_post = request_mock.post(
'http://localhost/notifications-template-preview/precompiled-preview.png',
content=png_content,
headers={'X-pdf-page-count': '1'},
status_code=500
)
admin_request.get(
'template.preview_letter_template_by_notification_id',
service_id=notification.service_id,
notification_id=notification.id,
file_type='png',
_expected_status=500
)
with pytest.raises(ValueError):
mock_post.last_request.json()
def test_preview_letter_template_precompiled_png_template_preview_400_error(
notify_api,
client,
admin_request,
sample_service,
mocker
):
template = create_template(sample_service,
template_type='letter',
template_name='Pre-compiled PDF',
subject='Pre-compiled PDF',
hidden=True)
notification = create_notification(template)
with set_config_values(notify_api, {
'TEMPLATE_PREVIEW_API_HOST': 'http://localhost/notifications-template-preview',
'TEMPLATE_PREVIEW_API_KEY': 'test-key'
}):
with requests_mock.Mocker() as request_mock:
pdf_content = b'\x00\x01'
png_content = b'\x00\x02'
mocker.patch('app.template.rest.get_letter_pdf', return_value=pdf_content)
mock_post = request_mock.post(
'http://localhost/notifications-template-preview/precompiled-preview.png',
content=png_content,
headers={'X-pdf-page-count': '1'},
status_code=404
)
admin_request.get(
'template.preview_letter_template_by_notification_id',
service_id=notification.service_id,
notification_id=notification.id,
file_type='png',
_expected_status=500
)
with pytest.raises(ValueError):
mock_post.last_request.json()
tests/app/test_commands.py
+14 -1
View File
@@ -1,6 +1,6 @@
from datetime import datetime
from app.commands import backfill_processing_time
from app.commands import backfill_performance_platform_totals, backfill_processing_time
def test_backfill_processing_time_works_for_correct_dates(mocker, notify_api):
@@ -14,3 +14,16 @@ def test_backfill_processing_time_works_for_correct_dates(mocker, notify_api):
send_mock.assert_any_call(datetime(2017, 7, 31, 23, 0), datetime(2017, 8, 1, 23, 0))
send_mock.assert_any_call(datetime(2017, 8, 1, 23, 0), datetime(2017, 8, 2, 23, 0))
send_mock.assert_any_call(datetime(2017, 8, 2, 23, 0), datetime(2017, 8, 3, 23, 0))
def test_backfill_totals_works_for_correct_dates(mocker, notify_api):
send_mock = mocker.patch('app.commands.send_total_sent_notifications_to_performance_platform')
# backfill_processing_time is a click.Command object - if you try invoking the callback on its own, it
# throws a `RuntimeError: There is no active click context.` - so get at the original function using __wrapped__
backfill_performance_platform_totals.callback.__wrapped__(datetime(2017, 8, 1), datetime(2017, 8, 3))
assert send_mock.call_count == 3
send_mock.assert_any_call(datetime(2017, 8, 1))
send_mock.assert_any_call(datetime(2017, 8, 2))
send_mock.assert_any_call(datetime(2017, 8, 3))
tests/app/test_model.py
+22 -1
View File
@@ -18,7 +18,8 @@ from app.models import (
NOTIFICATION_STATUS_LETTER_ACCEPTED,
NOTIFICATION_STATUS_LETTER_RECEIVED,
NOTIFICATION_STATUS_TYPES_FAILED,
NOTIFICATION_TECHNICAL_FAILURE
NOTIFICATION_TECHNICAL_FAILURE,
PRECOMPILED_TEMPLATE_NAME
)
from tests.app.conftest import (
sample_template as create_sample_template,
@@ -319,3 +320,23 @@ def test_letter_notification_postcode_can_be_null_for_precompiled_letters(client
assert json['line_1'] == 'test'
assert json['line_2'] == 'London'
assert json['postcode'] is None
def test_is_precompiled_letter_false(sample_letter_template):
assert not sample_letter_template.is_precompiled_letter
def test_is_precompiled_letter_true(sample_letter_template):
sample_letter_template.hidden = True
sample_letter_template.name = PRECOMPILED_TEMPLATE_NAME
assert sample_letter_template.is_precompiled_letter
def test_is_precompiled_letter_hidden_true_not_name(sample_letter_template):
sample_letter_template.hidden = True
assert not sample_letter_template.is_precompiled_letter
def test_is_precompiled_letter_name_correct_not_hidden(sample_letter_template):
sample_letter_template.name = PRECOMPILED_TEMPLATE_NAME
assert not sample_letter_template.is_precompiled_letter