diff --git a/README.md b/README.md index b85fe6479..11db3cb26 100644 --- a/README.md +++ b/README.md @@ -115,3 +115,30 @@ cf run-task notify-api "flask command purge_functional_test_data -u make cf-push` + +Once this is done, you can push your deployment changes to jenkins to have your app deployed on every deployment. diff --git a/app/celery/scheduled_tasks.py b/app/celery/scheduled_tasks.py index 9d5249e8a..b5f904f2b 100644 --- a/app/celery/scheduled_tasks.py +++ b/app/celery/scheduled_tasks.py @@ -213,19 +213,21 @@ def timeout_notifications(): @statsd(namespace="tasks") def send_daily_performance_platform_stats(): if performance_platform_client.active: - send_total_sent_notifications_to_performance_platform() + yesterday = datetime.utcnow() - timedelta(days=1) + send_total_sent_notifications_to_performance_platform(yesterday) processing_time.send_processing_time_to_performance_platform() -def send_total_sent_notifications_to_performance_platform(): - count_dict = total_sent_notifications.get_total_sent_notifications_yesterday() +def send_total_sent_notifications_to_performance_platform(day): + count_dict = total_sent_notifications.get_total_sent_notifications_for_day(day) email_sent_count = count_dict.get('email').get('count') sms_sent_count = count_dict.get('sms').get('count') + letter_sent_count = count_dict.get('letter').get('count') start_date = count_dict.get('start_date') current_app.logger.info( - "Attempting to update performance platform for date {} with email count {} and sms count {}" - .format(start_date, email_sent_count, sms_sent_count) + "Attempting to update Performance Platform for {} with {} emails, {} text messages and {} letters" + .format(start_date, email_sent_count, sms_sent_count, letter_sent_count) ) total_sent_notifications.send_total_notifications_sent_for_day_stats( @@ -240,6 +242,12 @@ def send_total_sent_notifications_to_performance_platform(): email_sent_count ) + total_sent_notifications.send_total_notifications_sent_for_day_stats( + start_date, + 'letter', + letter_sent_count + ) + @notify_celery.task(name='switch-current-sms-provider-on-slow-delivery') @statsd(namespace="tasks") diff --git a/app/celery/service_callback_tasks.py b/app/celery/service_callback_tasks.py index d1179ff17..21e87266e 100644 --- a/app/celery/service_callback_tasks.py +++ b/app/celery/service_callback_tasks.py @@ -6,6 +6,7 @@ from app import ( db, DATETIME_FORMAT, notify_celery, + encryption ) from app.dao.notifications_dao import ( get_notification_by_id, @@ -23,29 +24,83 @@ from app.config import QueueNames @notify_celery.task(bind=True, name="send-delivery-status", max_retries=5, default_retry_delay=300) @statsd(namespace="tasks") -def send_delivery_status_to_service(self, notification_id): - # TODO: do we need to do rate limit this? - notification = get_notification_by_id(notification_id) - service_callback_api = get_service_callback_api_for_service(service_id=notification.service_id) - if not service_callback_api: - # No delivery receipt API info set - return +def send_delivery_status_to_service(self, notification_id, + encrypted_status_update=None + ): + if not encrypted_status_update: + process_update_with_notification_id(self, notification_id=notification_id) + else: + try: + status_update = encryption.decrypt(encrypted_status_update) - # Release DB connection before performing an external HTTP request - db.session.close() + data = { + "id": str(notification_id), + "reference": status_update['notification_client_reference'], + "to": status_update['notification_to'], + "status": status_update['notification_status'], + "created_at": status_update['notification_created_at'], + "completed_at": status_update['notification_updated_at'], + "sent_at": status_update['notification_sent_at'], + "notification_type": status_update['notification_type'] + } - data = { - "id": str(notification_id), - "reference": str(notification.client_reference), - "to": notification.to, - "status": notification.status, - "created_at": notification.created_at.strftime(DATETIME_FORMAT), # the time service sent the request - "completed_at": notification.updated_at.strftime(DATETIME_FORMAT), # the last time the status was updated - "sent_at": notification.sent_at.strftime(DATETIME_FORMAT), # the time the email was sent - "notification_type": notification.notification_type - } + response = request( + method="POST", + url=status_update['service_callback_api_url'], + data=json.dumps(data), + headers={ + 'Content-Type': 'application/json', + 'Authorization': 'Bearer {}'.format(status_update['service_callback_api_bearer_token']) + }, + timeout=60 + ) + current_app.logger.info('send_delivery_status_to_service sending {} to {}, response {}'.format( + notification_id, + status_update['service_callback_api_url'], + response.status_code + )) + response.raise_for_status() + except RequestException as e: + current_app.logger.warning( + "send_delivery_status_to_service request failed for service_id: {} and url: {}. exc: {}".format( + notification_id, + status_update['service_callback_api_url'], + e + ) + ) + if not isinstance(e, HTTPError) or e.response.status_code >= 500: + try: + self.retry(queue=QueueNames.RETRY) + except self.MaxRetriesExceededError: + current_app.logger.exception( + """Retry: send_delivery_status_to_service has retried the max num of times + for notification: {}""".format(notification_id) + ) + +def process_update_with_notification_id(self, notification_id): + retry = False try: + notification = get_notification_by_id(notification_id) + service_callback_api = get_service_callback_api_for_service(service_id=notification.service_id) + if not service_callback_api: + # No delivery receipt API info set + return + + # Release DB connection before performing an external HTTP request + db.session.close() + + data = { + "id": str(notification_id), + "reference": str(notification.client_reference), + "to": notification.to, + "status": notification.status, + "created_at": notification.created_at.strftime(DATETIME_FORMAT), + "completed_at": notification.updated_at.strftime(DATETIME_FORMAT), + "sent_at": notification.sent_at.strftime(DATETIME_FORMAT), + "notification_type": notification.notification_type + } + response = request( method="POST", url=service_callback_api.url, @@ -71,7 +126,18 @@ def send_delivery_status_to_service(self, notification_id): ) ) if not isinstance(e, HTTPError) or e.response.status_code >= 500: - try: - self.retry(queue=QueueNames.RETRY) - except self.MaxRetriesExceededError: - current_app.logger.exception('Retry: send_delivery_status_to_service has retried the max num of times') + retry = True + except Exception as e: + current_app.logger.exception( + 'Unhandled exception when sending callback for notification {}'.format(notification_id) + ) + retry = True + + if retry: + try: + self.retry(queue=QueueNames.RETRY) + except self.MaxRetriesExceededError: + current_app.logger.exception( + """Retry: send_delivery_status_to_service has retried the max num of times + for notification: {}""".format(notification_id) + ) diff --git a/app/celery/statistics_tasks.py b/app/celery/statistics_tasks.py index 141f829b0..df6774f71 100644 --- a/app/celery/statistics_tasks.py +++ b/app/celery/statistics_tasks.py @@ -10,20 +10,9 @@ from app.dao.statistics_dao import ( update_job_stats_outcome_count ) from app.dao.notifications_dao import get_notification_by_id -from app.models import NOTIFICATION_STATUS_TYPES_COMPLETED from app.config import QueueNames -def create_initial_notification_statistic_tasks(notification): - if notification.job_id and notification.status: - record_initial_job_statistics.apply_async((str(notification.id),), queue=QueueNames.STATISTICS) - - -def create_outcome_notification_statistic_tasks(notification): - if notification.job_id and notification.status in NOTIFICATION_STATUS_TYPES_COMPLETED: - record_outcome_job_statistics.apply_async((str(notification.id),), queue=QueueNames.STATISTICS) - - @worker_process_shutdown.connect def worker_process_shutdown(sender, signal, pid, exitcode): current_app.logger.info('Statistics worker shutdown: PID: {} Exitcode: {}'.format(pid, exitcode)) diff --git a/app/celery/tasks.py b/app/celery/tasks.py index 20b3088c2..f317c634b 100644 --- a/app/celery/tasks.py +++ b/app/celery/tasks.py @@ -1,6 +1,6 @@ import json from datetime import datetime -from collections import namedtuple +from collections import namedtuple, defaultdict from celery.signals import worker_process_shutdown from flask import current_app @@ -31,6 +31,7 @@ from app import ( from app.aws import s3 from app.celery import provider_tasks, letters_pdf_tasks, research_mode_tasks from app.config import QueueNames +from app.dao.daily_sorted_letter_dao import dao_create_or_update_daily_sorted_letter from app.dao.inbound_sms_dao import dao_get_inbound_sms_by_id from app.dao.jobs_dao import ( dao_update_job, @@ -66,9 +67,11 @@ from app.models import ( NOTIFICATION_TEMPORARY_FAILURE, NOTIFICATION_TECHNICAL_FAILURE, SMS_TYPE, + DailySortedLetter, ) from app.notifications.process_notifications import persist_notification from app.service.utils import service_allowed_to_send_to +from app.utils import convert_utc_to_bst @worker_process_shutdown.connect @@ -404,6 +407,7 @@ def get_template_class(template_type): def update_letter_notifications_statuses(self, filename): bucket_location = '{}-ftp'.format(current_app.config['NOTIFY_EMAIL_DOMAIN']) response_file_content = s3.get_s3_file(bucket_location, filename) + sorted_letter_counts = defaultdict(int) try: notification_updates = process_updates_from_file(response_file_content) @@ -414,6 +418,7 @@ def update_letter_notifications_statuses(self, filename): for update in notification_updates: check_billable_units(update) update_letter_notification(filename, temporary_failures, update) + sorted_letter_counts[update.cost_threshold] += 1 if temporary_failures: # This will alert Notify that DVLA was unable to deliver the letters, we need to investigate @@ -421,6 +426,32 @@ def update_letter_notifications_statuses(self, filename): filename=filename, failures=temporary_failures) raise DVLAException(message) + if sorted_letter_counts.keys() - {'Unsorted', 'Sorted'}: + unknown_status = sorted_letter_counts.keys() - {'Unsorted', 'Sorted'} + + message = 'DVLA response file: {} contains unknown Sorted status {}'.format( + filename, unknown_status + ) + raise DVLAException(message) + + billing_date = get_billing_date_in_bst_from_filename(filename) + persist_daily_sorted_letter_counts(billing_date, sorted_letter_counts) + + +def get_billing_date_in_bst_from_filename(filename): + datetime_string = filename.split('.')[1] + datetime_obj = datetime.strptime(datetime_string, '%Y%m%d%H%M%S') + return convert_utc_to_bst(datetime_obj).date() + + +def persist_daily_sorted_letter_counts(day, sorted_letter_counts): + daily_letter_count = DailySortedLetter( + billing_day=day, + unsorted_count=sorted_letter_counts['Unsorted'], + sorted_count=sorted_letter_counts['Sorted'] + ) + dao_create_or_update_daily_sorted_letter(daily_letter_count) + def process_updates_from_file(response_file): NotificationUpdate = namedtuple('NotificationUpdate', ['reference', 'status', 'page_count', 'cost_threshold']) diff --git a/app/commands.py b/app/commands.py index 06a9b7a75..23fc18c0a 100644 --- a/app/commands.py +++ b/app/commands.py @@ -1,28 +1,33 @@ +import functools import uuid from datetime import datetime, timedelta from decimal import Decimal -import functools -import flask -from flask import current_app import click +import flask from click_datetime import Datetime as click_dt +from flask import current_app +from sqlalchemy.orm.exc import NoResultFound -from app import db +from app import db, DATETIME_FORMAT, encryption +from app.celery.scheduled_tasks import send_total_sent_notifications_to_performance_platform +from app.celery.service_callback_tasks import send_delivery_status_to_service +from app.config import QueueNames from app.dao.monthly_billing_dao import ( create_or_update_monthly_billing, get_monthly_billing_by_notification_type, get_service_ids_that_need_billing_populated ) -from app.models import PROVIDERS, User, SMS_TYPE, EMAIL_TYPE +from app.dao.provider_rates_dao import create_provider_rates as dao_create_provider_rates +from app.dao.service_callback_api_dao import get_service_callback_api_for_service from app.dao.services_dao import ( delete_service_and_all_associated_db_objects, dao_fetch_all_services_by_user ) -from app.dao.provider_rates_dao import create_provider_rates as dao_create_provider_rates from app.dao.users_dao import (delete_model_user, delete_user_verify_codes) +from app.models import PROVIDERS, User, SMS_TYPE, EMAIL_TYPE, Notification +from app.performance_platform.processing_time import (send_processing_time_for_start_and_end) from app.utils import get_midnight_for_day_before, get_london_midnight_in_utc -from app.performance_platform.processing_time import send_processing_time_for_start_and_end @click.group(name='command', help='Additional commands') @@ -209,12 +214,38 @@ def populate_monthly_billing(year): populate(service_id, year, i) +@notify_command() +@click.option('-s', '--start_date', required=True, help="start date inclusive", type=click_dt(format='%Y-%m-%d')) +@click.option('-e', '--end_date', required=True, help="end date inclusive", type=click_dt(format='%Y-%m-%d')) +def backfill_performance_platform_totals(start_date, end_date): + """ + Send historical total messages sent to Performance Platform. + + WARNING: This does not overwrite existing data. You need to delete + the existing data or Performance Platform will double-count. + """ + + delta = end_date - start_date + + print('Sending total messages sent for all days between {} and {}'.format(start_date, end_date)) + + for i in range(delta.days + 1): + + process_date = start_date + timedelta(days=i) + + print('Sending total messages sent for {}'.format( + process_date.isoformat() + )) + + send_total_sent_notifications_to_performance_platform(process_date) + + @notify_command() @click.option('-s', '--start_date', required=True, help="start date inclusive", type=click_dt(format='%Y-%m-%d')) @click.option('-e', '--end_date', required=True, help="end date inclusive", type=click_dt(format='%Y-%m-%d')) def backfill_processing_time(start_date, end_date): """ - Send historical performance platform stats. + Send historical processing time to Performance Platform. """ delta = end_date - start_date @@ -284,5 +315,55 @@ def insert_inbound_numbers_from_file(file_name): file.close() +@notify_command(name='replay-service-callbacks') +@click.option('-f', '--file_name', required=True, + help="""Full path of the file to upload, file is a contains client references of + notifications that need the status to be sent to the service.""") +@click.option('-s', '--service_id', required=True, + help="""The service that the callbacks are for""") +def replay_service_callbacks(file_name, service_id): + print("Start send service callbacks for service: ", service_id) + callback_api = get_service_callback_api_for_service(service_id=service_id) + if not callback_api: + print("Callback api was not found for service: {}".format(service_id)) + return + + errors = [] + notifications = [] + file = open(file_name) + + for ref in file: + try: + notification = Notification.query.filter_by(client_reference=ref.strip()).one() + notifications.append(notification) + except NoResultFound as e: + errors.append("Reference: {} was not found in notifications.".format(ref)) + + for e in errors: + print(e) + if errors: + raise Exception("Some notifications for the given references were not found") + + for n in notifications: + data = { + "notification_id": str(n.id), + "notification_client_reference": n.client_reference, + "notification_to": n.to, + "notification_status": n.status, + "notification_created_at": n.created_at.strftime(DATETIME_FORMAT), + "notification_updated_at": n.updated_at.strftime(DATETIME_FORMAT), + "notification_sent_at": n.sent_at.strftime(DATETIME_FORMAT), + "notification_type": n.notification_type, + "service_callback_api_url": callback_api.url, + "service_callback_api_bearer_token": callback_api.bearer_token, + } + encrypted_status_update = encryption.encrypt(data) + send_delivery_status_to_service.apply_async([str(n.id), encrypted_status_update], + queue=QueueNames.CALLBACKS) + + print("Replay service status for service: {}. Sent {} notification status updates to the queue".format( + service_id, len(notifications))) + + def setup_commands(application): application.cli.add_command(command_group) diff --git a/app/config.py b/app/config.py index f9b125be1..b1cdd93c2 100644 --- a/app/config.py +++ b/app/config.py @@ -230,11 +230,6 @@ class Config(object): 'schedule': crontab(hour=4, minute=40), 'options': {'queue': QueueNames.PERIODIC} }, - 'timeout-job-statistics': { - 'task': 'timeout-job-statistics', - 'schedule': crontab(hour=5, minute=0), - 'options': {'queue': QueueNames.PERIODIC} - }, 'populate_monthly_billing': { 'task': 'populate_monthly_billing', 'schedule': crontab(hour=5, minute=10), diff --git a/app/dao/daily_sorted_letter_dao.py b/app/dao/daily_sorted_letter_dao.py new file mode 100644 index 000000000..3afad4b2a --- /dev/null +++ b/app/dao/daily_sorted_letter_dao.py @@ -0,0 +1,37 @@ +from datetime import datetime + +from sqlalchemy.dialects.postgresql import insert + +from app import db +from app.dao.dao_utils import transactional +from app.models import DailySortedLetter + + +def dao_get_daily_sorted_letter_by_billing_day(billing_day): + return DailySortedLetter.query.filter_by( + billing_day=billing_day + ).first() + + +@transactional +def dao_create_or_update_daily_sorted_letter(new_daily_sorted_letter): + ''' + This uses the Postgres upsert to avoid race conditions when two threads try and insert + at the same row. The excluded object refers to values that we tried to insert but were + rejected. + http://docs.sqlalchemy.org/en/latest/dialects/postgresql.html#insert-on-conflict-upsert + ''' + table = DailySortedLetter.__table__ + stmt = insert(table).values( + billing_day=new_daily_sorted_letter.billing_day, + unsorted_count=new_daily_sorted_letter.unsorted_count, + sorted_count=new_daily_sorted_letter.sorted_count) + stmt = stmt.on_conflict_do_update( + index_elements=[table.c.billing_day], + set_={ + 'unsorted_count': table.c.unsorted_count + stmt.excluded.unsorted_count, + 'sorted_count': table.c.sorted_count + stmt.excluded.sorted_count, + 'updated_at': datetime.utcnow() + } + ) + db.session.connection().execute(stmt) diff --git a/app/dao/notifications_dao.py b/app/dao/notifications_dao.py index 155616801..7c25f8d21 100644 --- a/app/dao/notifications_dao.py +++ b/app/dao/notifications_dao.py @@ -83,6 +83,7 @@ def dao_get_template_usage(service_id, limit_days=None): Template.id.label('template_id'), Template.name, Template.template_type, + Template.is_precompiled_letter, notifications_aggregate_query.c.count ).join( notifications_aggregate_query, diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 57d5aa866..838a6336e 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -522,6 +522,7 @@ def dao_fetch_monthly_historical_usage_by_template_for_service(service_id, year) stat.month = result.month stat.year = result.year stat.count = result.count + stat.is_precompiled_letter = result.is_precompiled_letter stats.append(stat) month = get_london_month_from_utc_column(Notification.created_at) @@ -533,6 +534,7 @@ def dao_fetch_monthly_historical_usage_by_template_for_service(service_id, year) if fy_start < datetime.now() < fy_end: today_results = db.session.query( Notification.template_id, + Template.is_precompiled_letter, Template.name, Template.template_type, extract('month', month).label('month'), @@ -547,6 +549,7 @@ def dao_fetch_monthly_historical_usage_by_template_for_service(service_id, year) Notification.key_type != KEY_TYPE_TEST ).group_by( Notification.template_id, + Template.hidden, Template.name, Template.template_type, month, @@ -571,6 +574,7 @@ def dao_fetch_monthly_historical_usage_by_template_for_service(service_id, year) new_stat.month = int(today_result.month) new_stat.year = int(today_result.year) new_stat.count = today_result.count + new_stat.is_precompiled_letter = today_result.is_precompiled_letter stats.append(new_stat) return stats diff --git a/app/dao/stats_template_usage_by_month_dao.py b/app/dao/stats_template_usage_by_month_dao.py index 29aba1cdc..541ab7193 100644 --- a/app/dao/stats_template_usage_by_month_dao.py +++ b/app/dao/stats_template_usage_by_month_dao.py @@ -37,6 +37,7 @@ def dao_get_template_usage_stats_by_service(service_id, year): StatsTemplateUsageByMonth.template_id, Template.name, Template.template_type, + Template.is_precompiled_letter, StatsTemplateUsageByMonth.month, StatsTemplateUsageByMonth.year, StatsTemplateUsageByMonth.count diff --git a/app/dao/templates_dao.py b/app/dao/templates_dao.py index fb7dc7602..9dd397ef0 100644 --- a/app/dao/templates_dao.py +++ b/app/dao/templates_dao.py @@ -5,7 +5,11 @@ from sqlalchemy import asc, desc from sqlalchemy.sql.expression import bindparam from app import db -from app.models import (Template, TemplateHistory, TemplateRedacted) +from app.models import ( + Template, + TemplateHistory, + TemplateRedacted +) from app.dao.dao_utils import ( transactional, version_class @@ -135,6 +139,7 @@ def dao_get_templates_for_cache(cache): query = db.session.query(Template.id.label('template_id'), Template.template_type, Template.name, + Template.is_precompiled_letter, cache_subq.c.count.label('count') ).join(cache_subq, Template.id == cache_subq.c.template_id diff --git a/app/delivery/send_to_providers.py b/app/delivery/send_to_providers.py index b272f116b..4c1a3600b 100644 --- a/app/delivery/send_to_providers.py +++ b/app/delivery/send_to_providers.py @@ -31,7 +31,6 @@ from app.models import ( NOTIFICATION_SENT, NOTIFICATION_SENDING ) -from app.celery.statistics_tasks import create_initial_notification_statistic_tasks def send_sms_to_provider(notification): @@ -83,8 +82,6 @@ def send_sms_to_provider(notification): notification.billable_units = template.fragment_count update_notification(notification, provider, notification.international) - create_initial_notification_statistic_tasks(notification) - current_app.logger.debug( "SMS {} sent to provider {} at {}".format(notification.id, provider.get_name(), notification.sent_at) ) @@ -138,8 +135,6 @@ def send_email_to_provider(notification): notification.reference = reference update_notification(notification, provider) - create_initial_notification_statistic_tasks(notification) - current_app.logger.debug( "Email {} sent to provider at {}".format(notification.id, notification.sent_at) ) diff --git a/app/letters/utils.py b/app/letters/utils.py index 176944667..a7f05c00f 100644 --- a/app/letters/utils.py +++ b/app/letters/utils.py @@ -1,5 +1,6 @@ from datetime import datetime, timedelta +import boto3 from flask import current_app from notifications_utils.s3 import s3upload @@ -10,6 +11,8 @@ from app.variables import Retention LETTERS_PDF_FILE_LOCATION_STRUCTURE = \ '{folder}/NOTIFY.{reference}.{duplex}.{letter_class}.{colour}.{crown}.{date}.pdf' +PRECOMPILED_BUCKET_PREFIX = '{folder}/NOTIFY.{reference}' + def get_letter_pdf_filename(reference, crown): now = datetime.utcnow() @@ -31,6 +34,15 @@ def get_letter_pdf_filename(reference, crown): return upload_file_name +def get_bucket_prefix_for_notification(notification): + upload_file_name = PRECOMPILED_BUCKET_PREFIX.format( + folder=notification.created_at.date(), + reference=notification.reference + ).upper() + + return upload_file_name + + def upload_letter_pdf(notification, pdf_data): current_app.logger.info("PDF Letter {} reference {} created at {}, {} bytes".format( notification.id, notification.reference, notification.created_at, len(pdf_data))) @@ -48,3 +60,19 @@ def upload_letter_pdf(notification, pdf_data): current_app.logger.info("Uploaded letters PDF {} to {} for notification id {}".format( upload_file_name, current_app.config['LETTERS_PDF_BUCKET_NAME'], notification.id)) + + +def get_letter_pdf(notification): + bucket_name = current_app.config['LETTERS_PDF_BUCKET_NAME'] + + s3 = boto3.resource('s3') + bucket = s3.Bucket(bucket_name) + + for item in bucket.objects.filter(Prefix=get_bucket_prefix_for_notification(notification)): + obj = s3.Object( + bucket_name=bucket_name, + key=item.key + ) + file_content = obj.get()["Body"].read() + + return file_content diff --git a/app/models.py b/app/models.py index 6678d91e6..8c59ea7e9 100644 --- a/app/models.py +++ b/app/models.py @@ -6,6 +6,7 @@ from flask import url_for, current_app from sqlalchemy.ext.declarative import declared_attr from sqlalchemy.ext.associationproxy import association_proxy +from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.dialects.postgresql import ( UUID, JSON @@ -23,7 +24,7 @@ from notifications_utils.letter_timings import get_letter_timings from notifications_utils.template import ( PlainTextEmailTemplate, SMSMessageTemplate, - LetterDVLATemplate, + LetterPrintTemplate, ) from app.encryption import ( @@ -641,6 +642,9 @@ class TemplateProcessTypes(db.Model): name = db.Column(db.String(255), primary_key=True) +PRECOMPILED_TEMPLATE_NAME = 'Pre-compiled PDF' + + class TemplateBase(db.Model): __abstract__ = True @@ -718,6 +722,14 @@ class TemplateBase(db.Model): else: return None + @hybrid_property + def is_precompiled_letter(self): + return self.hidden and self.name == PRECOMPILED_TEMPLATE_NAME and self.template_type == LETTER_TYPE + + @is_precompiled_letter.setter + def is_precompiled_letter(self, value): + pass + def _as_utils_template(self): if self.template_type == EMAIL_TYPE: return PlainTextEmailTemplate( @@ -728,9 +740,8 @@ class TemplateBase(db.Model): {'content': self.content} ) if self.template_type == LETTER_TYPE: - return LetterDVLATemplate( + return LetterPrintTemplate( {'content': self.content, 'subject': self.subject}, - notification_reference=1, contact_block=self.service.get_default_letter_contact(), ) @@ -1752,3 +1763,13 @@ class StatsTemplateUsageByMonth(db.Model): 'year': self.year, 'count': self.count } + + +class DailySortedLetter(db.Model): + __tablename__ = "daily_sorted_letter" + + id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + billing_day = db.Column(db.Date, nullable=False, index=True, unique=True) + unsorted_count = db.Column(db.Integer, nullable=False, default=0) + sorted_count = db.Column(db.Integer, nullable=False, default=0) + updated_at = db.Column(db.DateTime, nullable=True, onupdate=datetime.datetime.utcnow) diff --git a/app/notifications/notifications_ses_callback.py b/app/notifications/notifications_ses_callback.py index cf3da7508..99909272a 100644 --- a/app/notifications/notifications_ses_callback.py +++ b/app/notifications/notifications_ses_callback.py @@ -11,7 +11,6 @@ from app.dao import ( notifications_dao ) from app.dao.service_callback_api_dao import get_service_callback_api_for_service -from app.celery.statistics_tasks import create_outcome_notification_statistic_tasks from app.notifications.process_client_response import validate_callback_data from app.celery.service_callback_tasks import send_delivery_status_to_service from app.config import QueueNames @@ -77,7 +76,6 @@ def process_ses_response(ses_request): notification.sent_at ) - create_outcome_notification_statistic_tasks(notification) _check_and_queue_callback_task(notification.id, notification.service_id) return diff --git a/app/notifications/process_client_response.py b/app/notifications/process_client_response.py index 19aa2d290..55380b0cc 100644 --- a/app/notifications/process_client_response.py +++ b/app/notifications/process_client_response.py @@ -8,7 +8,6 @@ from app.clients import ClientException from app.dao import notifications_dao from app.clients.sms.firetext import get_firetext_responses from app.clients.sms.mmg import get_mmg_responses -from app.celery.statistics_tasks import create_outcome_notification_statistic_tasks from app.celery.service_callback_tasks import send_delivery_status_to_service from app.config import QueueNames from app.dao.service_callback_api_dao import get_service_callback_api_for_service @@ -80,7 +79,6 @@ def _process_for_status(notification_status, client_name, reference): notification.sent_at ) - create_outcome_notification_statistic_tasks(notification) # queue callback task only if the service_callback_api exists service_callback_api = get_service_callback_api_for_service(service_id=notification.service_id) diff --git a/app/organisation/rest.py b/app/organisation/rest.py index 3bf3972b4..04100ce2a 100644 --- a/app/organisation/rest.py +++ b/app/organisation/rest.py @@ -107,3 +107,26 @@ def get_organisation_users(organisation_id): result = user_schema.dump(org_users, many=True) return jsonify(data=result.data) + + +@organisation_blueprint.route('/unique', methods=["GET"]) +def is_organisation_name_unique(): + organisation_id, name = check_request_args(request) + + name_exists = Organisation.query.filter(Organisation.name.ilike(name)).first() + + result = (not name_exists) or str(name_exists.id) == organisation_id + return jsonify(result=result), 200 + + +def check_request_args(request): + org_id = request.args.get('org_id') + name = request.args.get('name', None) + errors = [] + if not org_id: + errors.append({'org_id': ["Can't be empty"]}) + if not name: + errors.append({'name': ["Can't be empty"]}) + if errors: + raise InvalidRequest(errors, status_code=400) + return org_id, name diff --git a/app/performance_platform/total_sent_notifications.py b/app/performance_platform/total_sent_notifications.py index 4ad57171e..14695e9e9 100644 --- a/app/performance_platform/total_sent_notifications.py +++ b/app/performance_platform/total_sent_notifications.py @@ -1,11 +1,8 @@ -from datetime import datetime +from datetime import timedelta from app import performance_platform_client from app.dao.notifications_dao import get_total_sent_notifications_in_date_range -from app.utils import ( - get_london_midnight_in_utc, - get_midnight_for_day_before -) +from app.utils import get_london_midnight_in_utc def send_total_notifications_sent_for_day_stats(date, notification_type, count): @@ -20,13 +17,13 @@ def send_total_notifications_sent_for_day_stats(date, notification_type, count): performance_platform_client.send_stats_to_performance_platform(payload) -def get_total_sent_notifications_yesterday(): - today = datetime.utcnow() - start_date = get_midnight_for_day_before(today) - end_date = get_london_midnight_in_utc(today) +def get_total_sent_notifications_for_day(day): + start_date = get_london_midnight_in_utc(day) + end_date = start_date + timedelta(days=1) email_count = get_total_sent_notifications_in_date_range(start_date, end_date, 'email') sms_count = get_total_sent_notifications_in_date_range(start_date, end_date, 'sms') + letter_count = get_total_sent_notifications_in_date_range(start_date, end_date, 'letter') return { "start_date": start_date, @@ -35,5 +32,8 @@ def get_total_sent_notifications_yesterday(): }, "sms": { "count": sms_count - } + }, + "letter": { + "count": letter_count + }, } diff --git a/app/schemas.py b/app/schemas.py index 5592026b1..2131bb9b7 100644 --- a/app/schemas.py +++ b/app/schemas.py @@ -454,7 +454,16 @@ class NotificationWithTemplateSchema(BaseSchema): template = fields.Nested( TemplateSchema, - only=['id', 'version', 'name', 'template_type', 'content', 'subject', 'redact_personalisation'], + only=[ + 'id', + 'version', + 'name', + 'template_type', + 'content', + 'subject', + 'redact_personalisation', + 'is_precompiled_letter' + ], dump_only=True ) job = fields.Nested(JobSchema, only=["id", "original_file_name"], dump_only=True) diff --git a/app/service/rest.py b/app/service/rest.py index 99f03cee9..f6c7b8650 100644 --- a/app/service/rest.py +++ b/app/service/rest.py @@ -536,7 +536,8 @@ def get_monthly_template_usage(service_id): 'type': i.template_type, 'month': i.month, 'year': i.year, - 'count': i.count + 'count': i.count, + 'is_precompiled_letter': i.is_precompiled_letter } ) diff --git a/app/template/rest.py b/app/template/rest.py index ca30fa71d..c03a2c23e 100644 --- a/app/template/rest.py +++ b/app/template/rest.py @@ -1,4 +1,6 @@ import base64 + +import botocore from flask import ( Blueprint, current_app, @@ -18,6 +20,7 @@ from app.dao.templates_dao import ( dao_get_template_by_id) from notifications_utils.template import SMSMessageTemplate from app.dao.services_dao import dao_fetch_service_by_id +from app.letters.utils import get_letter_pdf from app.models import SMS_TYPE from app.notifications.validators import service_has_permission, check_reply_to from app.schemas import (template_schema, template_history_schema) @@ -29,7 +32,6 @@ from app.utils import get_template_instance, get_public_notify_type_text template_blueprint = Blueprint('template', __name__, url_prefix='/service//template') - register_errors(template_blueprint) @@ -194,36 +196,84 @@ def preview_letter_template_by_notification_id(service_id, notification_id, file template = dao_get_template_by_id(notification.template_id) - template_for_letter_print = { - "id": str(notification.template_id), - "subject": template.subject, - "content": template.content, - "version": str(template.version) - } + if template.is_precompiled_letter: - service = dao_fetch_service_by_id(service_id) + try: - data = { - 'letter_contact_block': notification.reply_to_text, - 'template': template_for_letter_print, - 'values': notification.personalisation, - 'dvla_org_id': service.dvla_organisation_id, - } + pdf_file = get_letter_pdf(notification) - resp = requests_post( - '{}/preview.{}{}'.format( + except botocore.exceptions.ClientError: + current_app.logger.exception( + 'Error getting letter file from S3 notification id {}'.format(notification_id)) + raise InvalidRequest('Error getting letter file from S3 notification id {}'.format(notification_id), + status_code=500) + + content = base64.b64encode(pdf_file).decode('utf-8') + + if file_type == 'png': + url = '{}/precompiled-preview.png{}'.format( + current_app.config['TEMPLATE_PREVIEW_API_HOST'], + '?page={}'.format(page) if page else '' + ) + + content = _get_png_preview(url, content, notification.id, json=False) + + else: + + template_for_letter_print = { + "id": str(notification.template_id), + "subject": template.subject, + "content": template.content, + "version": str(template.version) + } + + service = dao_fetch_service_by_id(service_id) + + data = { + 'letter_contact_block': notification.reply_to_text, + 'template': template_for_letter_print, + 'values': notification.personalisation, + 'dvla_org_id': service.dvla_organisation_id, + } + + url = '{}/preview.{}{}'.format( current_app.config['TEMPLATE_PREVIEW_API_HOST'], file_type, '?page={}'.format(page) if page else '' - ), - json=data, - headers={'Authorization': 'Token {}'.format(current_app.config['TEMPLATE_PREVIEW_API_KEY'])} - ) - - if resp.status_code != 200: - raise InvalidRequest( - 'Error generating preview for {}'.format(notification_id), status_code=500 ) - content = base64.b64encode(resp.content).decode('utf-8') + content = _get_png_preview(url, data, notification.id, json=True) + return jsonify({"content": content}) + + +def _get_png_preview(url, data, notification_id, json=True): + if json: + resp = requests_post( + url, + json=data, + headers={'Authorization': 'Token {}'.format(current_app.config['TEMPLATE_PREVIEW_API_KEY'])} + ) + else: + resp = requests_post( + url, + data=data, + headers={'Authorization': 'Token {}'.format(current_app.config['TEMPLATE_PREVIEW_API_KEY'])} + ) + + if resp.status_code != 200: + current_app.logger.exception( + 'Error generating preview letter for {} \nStatus code: {}\n{}'.format( + notification_id, + resp.status_code, + resp.content + )) + raise InvalidRequest( + 'Error generating preview letter for {}\nStatus code: {}\n{}'.format( + notification_id, + resp.status_code, + resp.content + ), status_code=500 + ) + + return base64.b64encode(resp.content).decode('utf-8') diff --git a/app/template_statistics/rest.py b/app/template_statistics/rest.py index 86548dc00..b7198409e 100644 --- a/app/template_statistics/rest.py +++ b/app/template_statistics/rest.py @@ -47,7 +47,8 @@ def get_template_statistics_for_service_by_day(service_id): 'count': data.count, 'template_id': str(data.template_id), 'template_name': data.name, - 'template_type': data.template_type + 'template_type': data.template_type, + 'is_precompiled_letter': data.is_precompiled_letter } return jsonify(data=[serialize(row) for row in stats]) diff --git a/docker/Dockerfile b/docker/Dockerfile index 2ccfe0e1e..bbafc0762 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -19,6 +19,8 @@ RUN \ build-essential \ zip \ libpq-dev \ + libffi-dev \ + python-dev \ jq \ && echo "Clean up" \ && rm -rf /var/lib/apt/lists/* /tmp/* diff --git a/manifest-delivery-base.yml b/manifest-delivery-base.yml index e141c6482..82bad9b2d 100644 --- a/manifest-delivery-base.yml +++ b/manifest-delivery-base.yml @@ -95,6 +95,6 @@ applications: NOTIFY_APP_NAME: delivery-worker-receipts - name: notify-delivery-worker-service-callbacks - command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO -P eventlet -c 1000 -Q service-callbacks + command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q service-callbacks env: - NOTIFY_APP_NAME: delivery-worker-service-callbacks + NOTIFY_APP_NAME: delivery-worker-service-callbacks \ No newline at end of file diff --git a/migrations/versions/0173_create_daily_sorted_letter.py b/migrations/versions/0173_create_daily_sorted_letter.py new file mode 100644 index 000000000..3215134b9 --- /dev/null +++ b/migrations/versions/0173_create_daily_sorted_letter.py @@ -0,0 +1,30 @@ +""" + +Revision ID: 0173_create_daily_sorted_letter +Revises: 0172_deprioritise_examples +Create Date: 2018-03-01 11:53:32.964256 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +revision = '0173_create_daily_sorted_letter' +down_revision = '0172_deprioritise_examples' + + +def upgrade(): + op.create_table('daily_sorted_letter', + sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('billing_day', sa.Date(), nullable=False), + sa.Column('unsorted_count', sa.Integer(), nullable=False), + sa.Column('sorted_count', sa.Integer(), nullable=False), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_daily_sorted_letter_billing_day'), 'daily_sorted_letter', ['billing_day'], unique=True) + + +def downgrade(): + op.drop_index(op.f('ix_daily_sorted_letter_billing_day'), table_name='daily_sorted_letter') + op.drop_table('daily_sorted_letter') diff --git a/requirements.txt b/requirements.txt index 0e06315c7..000ec0375 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,13 +16,13 @@ marshmallow==2.15.0 monotonic==1.4 psycopg2-binary==2.7.4 PyJWT==1.6.0 -SQLAlchemy==1.2.4 +SQLAlchemy==1.2.5 notifications-python-client==4.7.2 # PaaS awscli-cwlogs>=1.4,<1.5 -git+https://github.com/alphagov/notifications-utils.git@23.8.0#egg=notifications-utils==23.8.0 +git+https://github.com/alphagov/notifications-utils.git@24.0.0#egg=notifications-utils==24.0.0 git+https://github.com/alphagov/boto.git@2.43.0-patch3#egg=boto==2.43.0-patch3 diff --git a/requirements_for_test.txt b/requirements_for_test.txt index b41e1024d..f558aab7a 100644 --- a/requirements_for_test.txt +++ b/requirements_for_test.txt @@ -1,12 +1,12 @@ -r requirements.txt flake8==3.5.0 -pytest==3.4.1 +pytest==3.4.2 pytest-env==0.6.2 -pytest-mock==1.7.0 +pytest-mock==1.7.1 pytest-cov==2.5.1 pytest-xdist==1.22.2 -coveralls==1.2.0 -freezegun==0.3.9 +coveralls==1.3.0 +freezegun==0.3.10 requests-mock==1.4.0 # optional requirements for jsonschema strict-rfc3339==0.7 diff --git a/tests/app/celery/test_ftp_update_tasks.py b/tests/app/celery/test_ftp_update_tasks.py index 05b73329f..435ccab93 100644 --- a/tests/app/celery/test_ftp_update_tasks.py +++ b/tests/app/celery/test_ftp_update_tasks.py @@ -1,5 +1,5 @@ -from collections import namedtuple -from datetime import datetime +from collections import namedtuple, defaultdict +from datetime import datetime, date import pytest from freezegun import freeze_time @@ -18,12 +18,15 @@ from app.models import ( ) from app.celery.tasks import ( check_billable_units, + get_billing_date_in_bst_from_filename, + persist_daily_sorted_letter_counts, process_updates_from_file, update_dvla_job_to_error, update_letter_notifications_statuses, update_letter_notifications_to_error, update_letter_notifications_to_sent_to_dvla ) +from app.dao.daily_sorted_letter_dao import dao_get_daily_sorted_letter_by_billing_day from tests.app.db import create_notification, create_service_callback_api from tests.conftest import set_config @@ -56,8 +59,8 @@ def test_update_letter_notifications_statuses_raises_for_invalid_format(notify_a mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=invalid_file) with pytest.raises(DVLAException) as e: - update_letter_notifications_statuses(filename='foo.txt') - assert 'DVLA response file: {} has an invalid format'.format('foo.txt') in str(e) + update_letter_notifications_statuses(filename='NOTIFY.20170823160812.RSP.TXT') + assert 'DVLA response file: {} has an invalid format'.format('NOTIFY.20170823160812.RSP.TXT') in str(e) def test_update_letter_notification_statuses_when_notification_does_not_exist_updates_notification_history( @@ -70,7 +73,7 @@ def test_update_letter_notification_statuses_when_notification_does_not_exist_up billable_units=1) Notification.query.filter_by(id=notification.id).delete() - update_letter_notifications_statuses(filename="older_than_7_days.txt") + update_letter_notifications_statuses(filename="NOTIFY.20170823160812.RSP.TXT") updated_history = NotificationHistory.query.filter_by(id=notification.id).one() assert updated_history.status == NOTIFICATION_DELIVERED @@ -90,12 +93,35 @@ def test_update_letter_notifications_statuses_raises_dvla_exception(notify_api, ) in str(e) +def test_update_letter_notifications_statuses_raises_error_for_unknown_sorted_status( + notify_api, + mocker, + sample_letter_template +): + sent_letter_1 = create_notification(sample_letter_template, reference='ref-foo', status=NOTIFICATION_SENDING) + sent_letter_2 = create_notification(sample_letter_template, reference='ref-bar', status=NOTIFICATION_SENDING) + valid_file = '{}|Sent|1|Unsorted\n{}|Sent|2|Error'.format( + sent_letter_1.reference, sent_letter_2.reference) + + mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=valid_file) + + with pytest.raises(DVLAException) as e: + update_letter_notifications_statuses(filename='NOTIFY.20170823160812.RSP.TXT') + + assert "DVLA response file: {filename} contains unknown Sorted status {unknown_status}".format( + filename="NOTIFY.20170823160812.RSP.TXT", unknown_status="{'Error'}" + ) in str(e) + + def test_update_letter_notifications_statuses_calls_with_correct_bucket_location(notify_api, mocker): s3_mock = mocker.patch('app.celery.tasks.s3.get_s3_object') with set_config(notify_api, 'NOTIFY_EMAIL_DOMAIN', 'foo.bar'): - update_letter_notifications_statuses(filename='foo.txt') - s3_mock.assert_called_with('{}-ftp'.format(current_app.config['NOTIFY_EMAIL_DOMAIN']), 'foo.txt') + update_letter_notifications_statuses(filename='NOTIFY.20170823160812.RSP.TXT') + s3_mock.assert_called_with('{}-ftp'.format( + current_app.config['NOTIFY_EMAIL_DOMAIN']), + 'NOTIFY.20170823160812.RSP.TXT' + ) def test_update_letter_notifications_statuses_builds_updates_from_content(notify_api, mocker): @@ -103,7 +129,7 @@ def test_update_letter_notifications_statuses_builds_updates_from_content(notify mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=valid_file) update_mock = mocker.patch('app.celery.tasks.process_updates_from_file') - update_letter_notifications_statuses(filename='foo.txt') + update_letter_notifications_statuses(filename='NOTIFY.20170823160812.RSP.TXT') update_mock.assert_called_with('ref-foo|Sent|1|Unsorted\nref-bar|Sent|2|Sorted') @@ -136,7 +162,7 @@ def test_update_letter_notifications_statuses_persisted(notify_api, mocker, samp mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=valid_file) with pytest.raises(expected_exception=DVLAException) as e: - update_letter_notifications_statuses(filename='foo.txt') + update_letter_notifications_statuses(filename='NOTIFY.20170823160812.RSP.TXT') assert sent_letter.status == NOTIFICATION_DELIVERED assert sent_letter.billable_units == 1 @@ -145,7 +171,45 @@ def test_update_letter_notifications_statuses_persisted(notify_api, mocker, samp assert failed_letter.billable_units == 2 assert failed_letter.updated_at assert "DVLA response file: {filename} has failed letters with notification.reference {failures}".format( - filename="foo.txt", failures=[format(failed_letter.reference)]) in str(e) + filename="NOTIFY.20170823160812.RSP.TXT", failures=[format(failed_letter.reference)]) in str(e) + + +def test_update_letter_notifications_statuses_persists_daily_sorted_letter_count( + notify_api, + mocker, + sample_letter_template +): + sent_letter_1 = create_notification(sample_letter_template, reference='ref-foo', status=NOTIFICATION_SENDING) + sent_letter_2 = create_notification(sample_letter_template, reference='ref-bar', status=NOTIFICATION_SENDING) + valid_file = '{}|Sent|1|Unsorted\n{}|Sent|2|Sorted'.format( + sent_letter_1.reference, sent_letter_2.reference) + + mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=valid_file) + persist_letter_count_mock = mocker.patch('app.celery.tasks.persist_daily_sorted_letter_counts') + + update_letter_notifications_statuses(filename='NOTIFY.20170823160812.RSP.TXT') + + persist_letter_count_mock.assert_called_once_with(date(2017, 8, 23), {'Unsorted': 1, 'Sorted': 1}) + + +def test_update_letter_notifications_statuses_persists_daily_sorted_letter_count_with_no_sorted_values( + notify_api, + mocker, + sample_letter_template, + notify_db_session +): + sent_letter_1 = create_notification(sample_letter_template, reference='ref-foo', status=NOTIFICATION_SENDING) + sent_letter_2 = create_notification(sample_letter_template, reference='ref-bar', status=NOTIFICATION_SENDING) + valid_file = '{}|Sent|1|Unsorted\n{}|Sent|2|Unsorted'.format( + sent_letter_1.reference, sent_letter_2.reference) + mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=valid_file) + + update_letter_notifications_statuses(filename='NOTIFY.20170823160812.RSP.TXT') + + daily_sorted_letter = dao_get_daily_sorted_letter_by_billing_day(date(2017, 8, 23)) + + assert daily_sorted_letter.unsorted_count == 2 + assert daily_sorted_letter.sorted_count == 0 def test_update_letter_notifications_does_not_call_send_callback_if_no_db_entry(notify_api, mocker, @@ -159,7 +223,7 @@ def test_update_letter_notifications_does_not_call_send_callback_if_no_db_entry( 'app.celery.service_callback_tasks.send_delivery_status_to_service.apply_async' ) - update_letter_notifications_statuses(filename='foo.txt') + update_letter_notifications_statuses(filename='NOTIFY.20170823160812.RSP.TXT') send_mock.assert_not_called() @@ -230,3 +294,24 @@ def test_check_billable_units_when_billable_units_does_not_match_page_count( mock_logger.assert_called_once_with( 'Notification with id {} had 3 billable_units but a page count of 1'.format(notification.id) ) + + +@pytest.mark.parametrize('filename_date, billing_date', [ + ('20170820230000', date(2017, 8, 21)), + ('20170120230000', date(2017, 1, 20)) +]) +def test_get_billing_date_in_bst_from_filename(filename_date, billing_date): + filename = 'NOTIFY.{}.RSP.TXT'.format(filename_date) + result = get_billing_date_in_bst_from_filename(filename) + + assert result == billing_date + + +@freeze_time("2018-01-11 09:00:00") +def test_persist_daily_sorted_letter_counts_saves_sorted_and_unsorted_values(client, notify_db_session): + letter_counts = defaultdict(int, **{'Unsorted': 5, 'Sorted': 1}) + persist_daily_sorted_letter_counts(date.today(), letter_counts) + day = dao_get_daily_sorted_letter_by_billing_day(date.today()) + + assert day.unsorted_count == 5 + assert day.sorted_count == 1 diff --git a/tests/app/celery/test_scheduled_tasks.py b/tests/app/celery/test_scheduled_tasks.py index 6c2094cf8..8e3a3a33e 100644 --- a/tests/app/celery/test_scheduled_tasks.py +++ b/tests/app/celery/test_scheduled_tasks.py @@ -335,7 +335,7 @@ def test_send_total_sent_notifications_to_performance_platform_calls_with_correc new_callable=PropertyMock ) as mock_active: mock_active.return_value = True - send_total_sent_notifications_to_performance_platform() + send_total_sent_notifications_to_performance_platform(yesterday) perf_mock.assert_has_calls([ call(get_london_midnight_in_utc(yesterday), 'sms', 2), diff --git a/tests/app/celery/test_service_callback_tasks.py b/tests/app/celery/test_service_callback_tasks.py index a7c9aaeb4..11abf5f0f 100644 --- a/tests/app/celery/test_service_callback_tasks.py +++ b/tests/app/celery/test_service_callback_tasks.py @@ -1,39 +1,147 @@ +import uuid import json from datetime import datetime +from requests import RequestException import pytest import requests_mock +from sqlalchemy.exc import SQLAlchemyError -from requests import RequestException +from app import (DATETIME_FORMAT, encryption) -from app import (DATETIME_FORMAT) - -from tests.app.conftest import ( - sample_service as create_sample_service, - sample_template as create_sample_template, -) from tests.app.db import ( create_notification, - create_user, - create_service_callback_api + create_service_callback_api, + create_service, + create_template ) from app.celery.service_callback_tasks import send_delivery_status_to_service +from app.config import QueueNames @pytest.mark.parametrize("notification_type", ["email", "letter", "sms"]) -def test_send_delivery_status_to_service_post_https_request_to_service(notify_db, - notify_db_session, - notification_type): - user = create_user() - service = create_sample_service(notify_db, notify_db_session, user=user, restricted=True) +def test_send_delivery_status_to_service_post_https_request_to_service_with_encrypted_data( + notify_db_session, notification_type): + callback_api, template = _set_up_test_data(notification_type) + datestr = datetime(2017, 6, 20) + + notification = create_notification(template=template, + created_at=datestr, + updated_at=datestr, + sent_at=datestr, + status='sent' + ) + encrypted_status_update = _set_up_encrypted_data(callback_api, notification) + with requests_mock.Mocker() as request_mock: + request_mock.post(callback_api.url, + json={}, + status_code=200) + send_delivery_status_to_service(notification.id, encrypted_status_update=encrypted_status_update) + + mock_data = { + "id": str(notification.id), + "reference": notification.client_reference, + "to": notification.to, + "status": notification.status, + "created_at": datestr.strftime(DATETIME_FORMAT), + "completed_at": datestr.strftime(DATETIME_FORMAT), + "sent_at": datestr.strftime(DATETIME_FORMAT), + "notification_type": notification_type + } + + assert request_mock.call_count == 1 + assert request_mock.request_history[0].url == callback_api.url + assert request_mock.request_history[0].method == 'POST' + assert request_mock.request_history[0].text == json.dumps(mock_data) + assert request_mock.request_history[0].headers["Content-type"] == "application/json" + assert request_mock.request_history[0].headers["Authorization"] == "Bearer {}".format(callback_api.bearer_token) + + +@pytest.mark.parametrize("notification_type", + ["email", "letter", "sms"]) +def test_send_delivery_status_to_service_retries_if_request_returns_500_with_encrypted_data( + notify_db_session, mocker, notification_type +): + callback_api, template = _set_up_test_data(notification_type) + datestr = datetime(2017, 6, 20) + notification = create_notification(template=template, + created_at=datestr, + updated_at=datestr, + sent_at=datestr, + status='sent' + ) + encrypted_data = _set_up_encrypted_data(callback_api, notification) + mocked = mocker.patch('app.celery.service_callback_tasks.send_delivery_status_to_service.retry') + with requests_mock.Mocker() as request_mock: + request_mock.post(callback_api.url, + json={}, + status_code=500) + send_delivery_status_to_service(notification.id, encrypted_status_update=encrypted_data) + + assert mocked.call_count == 1 + assert mocked.call_args[1]['queue'] == 'retry-tasks' + + +@pytest.mark.parametrize("notification_type", + ["email", "letter", "sms"]) +def test_send_delivery_status_to_service_does_not_retries_if_request_returns_404_with_encrypted_data( + notify_db_session, + mocker, + notification_type +): + callback_api, template = _set_up_test_data(notification_type) + datestr = datetime(2017, 6, 20) + notification = create_notification(template=template, + created_at=datestr, + updated_at=datestr, + sent_at=datestr, + status='sent' + ) + encrypted_data = _set_up_encrypted_data(callback_api, notification) + mocked = mocker.patch('app.celery.service_callback_tasks.send_delivery_status_to_service.retry') + with requests_mock.Mocker() as request_mock: + request_mock.post(callback_api.url, + json={}, + status_code=404) + send_delivery_status_to_service(notification.id, encrypted_status_update=encrypted_data) + + assert mocked.call_count == 0 + + +def _set_up_test_data(notification_type): + service = create_service(restricted=True) + template = create_template(service=service, template_type=notification_type, subject='Hello') callback_api = create_service_callback_api(service=service, url="https://some.service.gov.uk/", bearer_token="something_unique") - template = create_sample_template( - notify_db, notify_db_session, service=service, template_type=notification_type, subject_line='Hello' - ) + return callback_api, template + +def _set_up_encrypted_data(callback_api, notification): + data = { + "notification_id": str(notification.id), + "notification_client_reference": notification.client_reference, + "notification_to": notification.to, + "notification_status": notification.status, + "notification_created_at": notification.created_at.strftime(DATETIME_FORMAT), + "notification_updated_at": notification.updated_at.strftime(DATETIME_FORMAT), + "notification_sent_at": notification.sent_at.strftime(DATETIME_FORMAT), + "notification_type": notification.notification_type, + "service_callback_api_url": callback_api.url, + "service_callback_api_bearer_token": callback_api.bearer_token, + } + encrypted_status_update = encryption.encrypt(data) + return encrypted_status_update + + +# We are updating the task to take everything it needs so that there are no db calls. +# The following tests will be deleted once that is complete. +@pytest.mark.parametrize("notification_type", + ["email", "letter", "sms"]) +def test_send_delivery_status_to_service_post_https_request_to_service( + notify_db_session, notification_type): + callback_api, template = _set_up_test_data(notification_type) datestr = datetime(2017, 6, 20) notification = create_notification(template=template, @@ -71,12 +179,9 @@ def test_send_delivery_status_to_service_post_https_request_to_service(notify_db @pytest.mark.parametrize("notification_type", ["email", "letter", "sms"]) def test_send_delivery_status_to_service_does_not_sent_request_when_service_callback_api_does_not_exist( - notify_db, notify_db_session, mocker, notification_type): - service = create_sample_service(notify_db, notify_db_session, restricted=True) - - template = create_sample_template( - notify_db, notify_db_session, service=service, template_type=notification_type, subject_line='Hello' - ) + notify_db_session, mocker, notification_type): + service = create_service(restricted=True) + template = create_template(service=service, template_type=notification_type, subject='Hello') datestr = datetime(2017, 6, 20) notification = create_notification(template=template, @@ -88,23 +193,15 @@ def test_send_delivery_status_to_service_does_not_sent_request_when_service_call mocked = mocker.patch("requests.request") send_delivery_status_to_service(notification.id) - mocked.call_count == 0 + assert mocked.call_count == 0 @pytest.mark.parametrize("notification_type", ["email", "letter", "sms"]) -def test_send_delivery_status_to_service_retries_if_request_returns_500(notify_db, - notify_db_session, +def test_send_delivery_status_to_service_retries_if_request_returns_500(notify_db_session, mocker, notification_type): - user = create_user() - service = create_sample_service(notify_db, notify_db_session, user=user, restricted=True) - - template = create_sample_template( - notify_db, notify_db_session, service=service, template_type=notification_type, subject_line='Hello' - ) - callback_api = create_service_callback_api(service=service, url="https://some.service.gov.uk/", - bearer_token="something_unique") + callback_api, template = _set_up_test_data(notification_type) datestr = datetime(2017, 6, 20) notification = create_notification(template=template, created_at=datestr, @@ -125,18 +222,11 @@ def test_send_delivery_status_to_service_retries_if_request_returns_500(notify_d @pytest.mark.parametrize("notification_type", ["email", "letter", "sms"]) -def test_send_delivery_status_to_service_retries_if_request_throws_unknown(notify_db, - notify_db_session, +def test_send_delivery_status_to_service_retries_if_request_throws_unknown(notify_db_session, mocker, notification_type): - user = create_user() - service = create_sample_service(notify_db, notify_db_session, user=user, restricted=True) - template = create_sample_template( - notify_db, notify_db_session, service=service, template_type=notification_type, subject_line='Hello' - ) - create_service_callback_api(service=service, url="https://some.service.gov.uk/", - bearer_token="something_unique") + callback_api, template = _set_up_test_data(notification_type) datestr = datetime(2017, 6, 20) notification = create_notification(template=template, created_at=datestr, @@ -156,18 +246,12 @@ def test_send_delivery_status_to_service_retries_if_request_throws_unknown(notif @pytest.mark.parametrize("notification_type", ["email", "letter", "sms"]) -def test_send_delivery_status_to_service_does_not_retries_if_request_returns_404(notify_db, - notify_db_session, - mocker, - notification_type): - user = create_user() - service = create_sample_service(notify_db, notify_db_session, user=user, restricted=True) - - template = create_sample_template( - notify_db, notify_db_session, service=service, template_type=notification_type, subject_line='Hello' - ) - callback_api = create_service_callback_api(service=service, url="https://some.service.gov.uk/", - bearer_token="something_unique") +def test_send_delivery_status_to_service_does_not_retries_if_request_returns_404( + notify_db_session, + mocker, + notification_type +): + callback_api, template = _set_up_test_data(notification_type) datestr = datetime(2017, 6, 20) notification = create_notification(template=template, created_at=datestr, @@ -182,4 +266,15 @@ def test_send_delivery_status_to_service_does_not_retries_if_request_returns_404 status_code=404) send_delivery_status_to_service(notification.id) - mocked.call_count == 0 + assert mocked.call_count == 0 + + +def test_send_delivery_status_to_service_retries_if_database_error(client, mocker): + notification_id = uuid.uuid4() + db_call = mocker.patch('app.celery.service_callback_tasks.get_notification_by_id', side_effect=SQLAlchemyError) + retry = mocker.patch('app.celery.service_callback_tasks.send_delivery_status_to_service.retry') + + send_delivery_status_to_service(notification_id) + + db_call.assert_called_once_with(notification_id) + retry.assert_called_once_with(queue=QueueNames.RETRY) diff --git a/tests/app/celery/test_statistics_tasks.py b/tests/app/celery/test_statistics_tasks.py deleted file mode 100644 index bb6c4c5b6..000000000 --- a/tests/app/celery/test_statistics_tasks.py +++ /dev/null @@ -1,169 +0,0 @@ -import pytest -from app.celery.statistics_tasks import ( - record_initial_job_statistics, - record_outcome_job_statistics, - create_initial_notification_statistic_tasks, - create_outcome_notification_statistic_tasks) -from sqlalchemy.exc import SQLAlchemyError -from app import create_uuid -from tests.app.conftest import sample_notification -from app.models import ( - NOTIFICATION_STATUS_TYPES_COMPLETED, - NOTIFICATION_SENDING, - NOTIFICATION_PENDING, - NOTIFICATION_CREATED, - NOTIFICATION_DELIVERED, -) - - -def test_should_create_initial_job_task_if_notification_is_related_to_a_job( - notify_db, notify_db_session, sample_job, mocker -): - mock = mocker.patch("app.celery.statistics_tasks.record_initial_job_statistics.apply_async") - notification = sample_notification(notify_db, notify_db_session, job=sample_job) - create_initial_notification_statistic_tasks(notification) - mock.assert_called_once_with((str(notification.id), ), queue="statistics-tasks") - - -@pytest.mark.parametrize('status', [ - NOTIFICATION_SENDING, NOTIFICATION_CREATED, NOTIFICATION_PENDING -]) -def test_should_create_intial_job_task_if_notification_is_not_in_completed_state( - notify_db, notify_db_session, sample_job, mocker, status -): - mock = mocker.patch("app.celery.statistics_tasks.record_initial_job_statistics.apply_async") - notification = sample_notification(notify_db, notify_db_session, job=sample_job, status=status) - create_initial_notification_statistic_tasks(notification) - mock.assert_called_once_with((str(notification.id), ), queue="statistics-tasks") - - -def test_should_not_create_initial_job_task_if_notification_is_not_related_to_a_job( - notify_db, notify_db_session, mocker -): - notification = sample_notification(notify_db, notify_db_session, status=NOTIFICATION_CREATED) - mock = mocker.patch("app.celery.statistics_tasks.record_initial_job_statistics.apply_async") - create_initial_notification_statistic_tasks(notification) - mock.assert_not_called() - - -def test_should_create_outcome_job_task_if_notification_is_related_to_a_job( - notify_db, notify_db_session, sample_job, mocker -): - mock = mocker.patch("app.celery.statistics_tasks.record_outcome_job_statistics.apply_async") - notification = sample_notification(notify_db, notify_db_session, job=sample_job, status=NOTIFICATION_DELIVERED) - create_outcome_notification_statistic_tasks(notification) - mock.assert_called_once_with((str(notification.id), ), queue="statistics-tasks") - - -@pytest.mark.parametrize('status', NOTIFICATION_STATUS_TYPES_COMPLETED) -def test_should_create_outcome_job_task_if_notification_is_in_completed_state( - notify_db, notify_db_session, sample_job, mocker, status -): - mock = mocker.patch("app.celery.statistics_tasks.record_outcome_job_statistics.apply_async") - notification = sample_notification(notify_db, notify_db_session, job=sample_job, status=status) - create_outcome_notification_statistic_tasks(notification) - mock.assert_called_once_with((str(notification.id), ), queue="statistics-tasks") - - -@pytest.mark.parametrize('status', [ - NOTIFICATION_SENDING, NOTIFICATION_CREATED, NOTIFICATION_PENDING -]) -def test_should_not_create_outcome_job_task_if_notification_is_not_in_completed_state_already( - notify_db, notify_db_session, sample_job, mocker, status -): - mock = mocker.patch("app.celery.statistics_tasks.record_initial_job_statistics.apply_async") - notification = sample_notification(notify_db, notify_db_session, job=sample_job, status=status) - create_outcome_notification_statistic_tasks(notification) - mock.assert_not_called() - - -def test_should_not_create_outcome_job_task_if_notification_is_not_related_to_a_job( - notify_db, notify_db_session, sample_notification, mocker -): - mock = mocker.patch("app.celery.statistics_tasks.record_outcome_job_statistics.apply_async") - create_outcome_notification_statistic_tasks(sample_notification) - mock.assert_not_called() - - -def test_should_call_create_job_stats_dao_methods(notify_db, notify_db_session, sample_notification, mocker): - dao_mock = mocker.patch("app.celery.statistics_tasks.create_or_update_job_sending_statistics") - record_initial_job_statistics(str(sample_notification.id)) - - dao_mock.assert_called_once_with(sample_notification) - - -def test_should_retry_if_persisting_the_job_stats_has_a_sql_alchemy_exception( - notify_db, - notify_db_session, - sample_notification, - mocker): - dao_mock = mocker.patch( - "app.celery.statistics_tasks.create_or_update_job_sending_statistics", - side_effect=SQLAlchemyError() - ) - retry_mock = mocker.patch('app.celery.statistics_tasks.record_initial_job_statistics.retry') - - record_initial_job_statistics(str(sample_notification.id)) - dao_mock.assert_called_once_with(sample_notification) - retry_mock.assert_called_with(queue="retry-tasks") - - -def test_should_call_update_job_stats_dao_outcome_methods(notify_db, notify_db_session, sample_notification, mocker): - dao_mock = mocker.patch("app.celery.statistics_tasks.update_job_stats_outcome_count") - record_outcome_job_statistics(str(sample_notification.id)) - - dao_mock.assert_called_once_with(sample_notification) - - -def test_should_retry_if_persisting_the_job_outcome_stats_has_a_sql_alchemy_exception( - notify_db, - notify_db_session, - sample_notification, - mocker): - dao_mock = mocker.patch( - "app.celery.statistics_tasks.update_job_stats_outcome_count", - side_effect=SQLAlchemyError() - ) - retry_mock = mocker.patch('app.celery.statistics_tasks.record_outcome_job_statistics.retry') - - record_outcome_job_statistics(str(sample_notification.id)) - dao_mock.assert_called_once_with(sample_notification) - retry_mock.assert_called_with(queue="retry-tasks") - - -def test_should_retry_if_persisting_the_job_outcome_stats_updates_zero_rows( - notify_db, - notify_db_session, - sample_notification, - mocker): - dao_mock = mocker.patch("app.celery.statistics_tasks.update_job_stats_outcome_count", return_value=0) - retry_mock = mocker.patch('app.celery.statistics_tasks.record_outcome_job_statistics.retry') - - record_outcome_job_statistics(str(sample_notification.id)) - dao_mock.assert_called_once_with(sample_notification) - retry_mock.assert_called_with(queue="retry-tasks") - - -def test_should_retry_if_persisting_the_job_stats_creation_cant_find_notification_by_id( - notify_db, - notify_db_session, - mocker): - dao_mock = mocker.patch("app.celery.statistics_tasks.create_or_update_job_sending_statistics") - retry_mock = mocker.patch('app.celery.statistics_tasks.record_initial_job_statistics.retry') - - record_initial_job_statistics(str(create_uuid())) - dao_mock.assert_not_called() - retry_mock.assert_called_with(queue="retry-tasks") - - -def test_should_retry_if_persisting_the_job_stats_outcome_cant_find_notification_by_id( - notify_db, - notify_db_session, - mocker): - - dao_mock = mocker.patch("app.celery.statistics_tasks.update_job_stats_outcome_count") - retry_mock = mocker.patch('app.celery.statistics_tasks.record_outcome_job_statistics.retry') - - record_outcome_job_statistics(str(create_uuid())) - dao_mock.assert_not_called() - retry_mock.assert_called_with(queue="retry-tasks") diff --git a/tests/app/celery/test_tasks.py b/tests/app/celery/test_tasks.py index 0ff66a4f7..f8e00651f 100644 --- a/tests/app/celery/test_tasks.py +++ b/tests/app/celery/test_tasks.py @@ -9,7 +9,7 @@ from freezegun import freeze_time from requests import RequestException from sqlalchemy.exc import SQLAlchemyError from celery.exceptions import Retry -from notifications_utils.template import SMSMessageTemplate, WithSubjectTemplate, LetterDVLATemplate +from notifications_utils.template import SMSMessageTemplate, WithSubjectTemplate from app import (encryption, DATETIME_FORMAT) from app.celery import provider_tasks @@ -1209,14 +1209,6 @@ def test_get_template_class(template_type, expected_class): assert get_template_class(template_type) == expected_class -@freeze_time("2017-03-23 11:09:00.061258") -def test_dvla_letter_template(sample_letter_notification): - t = {"content": sample_letter_notification.template.content, - "subject": sample_letter_notification.template.subject} - letter = LetterDVLATemplate(t, sample_letter_notification.personalisation, "random-string") - assert str(letter) == "140|500|001||random-string|||||||||||||A1||A2|A3|A4|A5|A6|A_POST|||||||||23 March 2017

Template subjectDear Sir/Madam, Hello. Yours Truly, The Government." # noqa - - def test_send_inbound_sms_to_service_post_https_request_to_service(notify_api, sample_service): inbound_api = create_service_inbound_api(service=sample_service, url="https://some.service.gov.uk/", bearer_token="something_unique") diff --git a/tests/app/dao/test_daily_sorted_letter_dao.py b/tests/app/dao/test_daily_sorted_letter_dao.py new file mode 100644 index 000000000..2a4ceb318 --- /dev/null +++ b/tests/app/dao/test_daily_sorted_letter_dao.py @@ -0,0 +1,47 @@ +from datetime import date + +from app.dao.daily_sorted_letter_dao import ( + dao_create_or_update_daily_sorted_letter, + dao_get_daily_sorted_letter_by_billing_day, +) +from app.models import DailySortedLetter +from tests.app.db import create_daily_sorted_letter + + +def test_dao_get_daily_sorted_letter_by_billing_day(notify_db, notify_db_session): + billing_day = date(2018, 2, 1) + other_day = date(2017, 9, 8) + + daily_sorted_letters = create_daily_sorted_letter(billing_day=billing_day) + + assert dao_get_daily_sorted_letter_by_billing_day(billing_day) == daily_sorted_letters + assert not dao_get_daily_sorted_letter_by_billing_day(other_day) + + +def test_dao_create_or_update_daily_sorted_letter_creates_a_new_entry(notify_db, notify_db_session): + billing_day = date(2018, 2, 1) + dsl = DailySortedLetter(billing_day=billing_day, unsorted_count=2, sorted_count=0) + dao_create_or_update_daily_sorted_letter(dsl) + + daily_sorted_letter = dao_get_daily_sorted_letter_by_billing_day(billing_day) + + assert daily_sorted_letter.billing_day == billing_day + assert daily_sorted_letter.unsorted_count == 2 + assert daily_sorted_letter.sorted_count == 0 + assert not daily_sorted_letter.updated_at + + +def test_dao_create_or_update_daily_sorted_letter_updates_an_existing_entry( + notify_db, + notify_db_session +): + create_daily_sorted_letter(unsorted_count=2, sorted_count=3) + + dsl = DailySortedLetter(billing_day=date(2018, 1, 18), unsorted_count=5, sorted_count=17) + dao_create_or_update_daily_sorted_letter(dsl) + + daily_sorted_letter = dao_get_daily_sorted_letter_by_billing_day(dsl.billing_day) + + assert daily_sorted_letter.unsorted_count == 7 + assert daily_sorted_letter.sorted_count == 20 + assert daily_sorted_letter.updated_at diff --git a/tests/app/dao/test_stats_template_usage_by_month_dao.py b/tests/app/dao/test_stats_template_usage_by_month_dao.py index 7d21696b5..676e00952 100644 --- a/tests/app/dao/test_stats_template_usage_by_month_dao.py +++ b/tests/app/dao/test_stats_template_usage_by_month_dao.py @@ -3,7 +3,7 @@ from app.dao.stats_template_usage_by_month_dao import ( insert_or_update_stats_for_template, dao_get_template_usage_stats_by_service ) -from app.models import StatsTemplateUsageByMonth +from app.models import StatsTemplateUsageByMonth, LETTER_TYPE, PRECOMPILED_TEMPLATE_NAME from tests.app.db import create_service, create_template @@ -74,6 +74,36 @@ def test_dao_get_template_usage_stats_by_service(sample_service): assert len(result) == 1 +def test_dao_get_template_usage_stats_by_service_for_precompiled_letters(sample_service): + + letter_template = create_template(service=sample_service, template_type=LETTER_TYPE) + + precompiled_letter_template = create_template( + service=sample_service, template_name=PRECOMPILED_TEMPLATE_NAME, hidden=True, template_type=LETTER_TYPE) + + db.session.add(StatsTemplateUsageByMonth( + template_id=letter_template.id, + month=5, + year=2017, + count=10 + )) + + db.session.add(StatsTemplateUsageByMonth( + template_id=precompiled_letter_template.id, + month=4, + year=2017, + count=20 + )) + + result = dao_get_template_usage_stats_by_service(sample_service.id, 2017) + + assert len(result) == 2 + assert [ + (letter_template.id, 'letter Template Name', 'letter', False, 5, 2017, 10), + (precompiled_letter_template.id, PRECOMPILED_TEMPLATE_NAME, 'letter', True, 4, 2017, 20) + ] == result + + def test_dao_get_template_usage_stats_by_service_specific_year(sample_service): email_template = create_template(service=sample_service, template_type="email") diff --git a/tests/app/dao/test_templates_dao.py b/tests/app/dao/test_templates_dao.py index 8891d2e26..d4768f598 100644 --- a/tests/app/dao/test_templates_dao.py +++ b/tests/app/dao/test_templates_dao.py @@ -13,7 +13,12 @@ from app.dao.templates_dao import ( dao_get_templates_for_cache, dao_redact_template, dao_update_template_reply_to ) -from app.models import Template, TemplateHistory, TemplateRedacted +from app.models import ( + Template, + TemplateHistory, + TemplateRedacted, + PRECOMPILED_TEMPLATE_NAME +) from tests.app.conftest import sample_template as create_sample_template from tests.app.db import create_template, create_letter_contact @@ -503,8 +508,33 @@ def test_get_templates_by_ids_successful(notify_db, notify_db_session): cache = [[k, v] for k, v in sample_cache_dict.items()] templates = dao_get_templates_for_cache(cache) assert len(templates) == 2 - assert [(template_1.id, template_1.template_type, template_1.name, 2), - (template_2.id, template_2.template_type, template_2.name, 3)] == templates + assert [(template_1.id, template_1.template_type, template_1.name, False, 2), + (template_2.id, template_2.template_type, template_2.name, False, 3)] == templates + + +def test_get_letter_templates_by_ids_successful(notify_db, notify_db_session): + template_1 = create_sample_template( + notify_db, + notify_db_session, + template_name=PRECOMPILED_TEMPLATE_NAME, + template_type="letter", + content="Template content", + hidden=True + ) + template_2 = create_sample_template( + notify_db, + notify_db_session, + template_name='Sample Template 2', + template_type="letter", + content="Template content" + ) + sample_cache_dict = {str.encode(str(template_1.id)): str.encode('2'), + str.encode(str(template_2.id)): str.encode('3')} + cache = [[k, v] for k, v in sample_cache_dict.items()] + templates = dao_get_templates_for_cache(cache) + assert len(templates) == 2 + assert [(template_1.id, template_1.template_type, template_1.name, True, 2), + (template_2.id, template_2.template_type, template_2.name, False, 3)] == templates def test_get_templates_by_ids_successful_for_one_cache_item(notify_db, notify_db_session): @@ -519,7 +549,7 @@ def test_get_templates_by_ids_successful_for_one_cache_item(notify_db, notify_db cache = [[k, v] for k, v in sample_cache_dict.items()] templates = dao_get_templates_for_cache(cache) assert len(templates) == 1 - assert [(template_1.id, template_1.template_type, template_1.name, 2)] == templates + assert [(template_1.id, template_1.template_type, template_1.name, False, 2)] == templates def test_get_templates_by_ids_returns_empty_list(): diff --git a/tests/app/db.py b/tests/app/db.py index 16ff29910..87c344d3e 100644 --- a/tests/app/db.py +++ b/tests/app/db.py @@ -1,4 +1,4 @@ -from datetime import datetime +from datetime import datetime, date import uuid from app import db @@ -9,6 +9,7 @@ from app.dao.service_sms_sender_dao import update_existing_sms_sender_with_inbou from app.dao.invited_org_user_dao import save_invited_org_user from app.models import ( ApiKey, + DailySortedLetter, InboundSms, InboundNumber, Job, @@ -130,7 +131,8 @@ def create_template( template_name=None, subject='Template subject', content='Dear Sir/Madam, Hello. Yours Truly, The Government.', - reply_to=None + reply_to=None, + hidden=False ): data = { 'name': template_name or '{} Template Name'.format(template_type), @@ -139,6 +141,7 @@ def create_template( 'service': service, 'created_by': service.created_by, 'reply_to': reply_to, + 'hidden': hidden } if template_type != SMS_TYPE: data['subject'] = subject @@ -504,3 +507,16 @@ def create_invited_org_user(organisation, invited_by, email_address='invite@exam ) save_invited_org_user(invited_org_user) return invited_org_user + + +def create_daily_sorted_letter(billing_day=date(2018, 1, 18), unsorted_count=0, sorted_count=0): + daily_sorted_letter = DailySortedLetter( + billing_day=billing_day, + unsorted_count=unsorted_count, + sorted_count=sorted_count + ) + + db.session.add(daily_sorted_letter) + db.session.commit() + + return daily_sorted_letter diff --git a/tests/app/delivery/test_send_to_providers.py b/tests/app/delivery/test_send_to_providers.py index 4b973975e..627768220 100644 --- a/tests/app/delivery/test_send_to_providers.py +++ b/tests/app/delivery/test_send_to_providers.py @@ -1,7 +1,7 @@ import uuid from collections import namedtuple from datetime import datetime -from unittest.mock import ANY, call +from unittest.mock import ANY import pytest from flask import current_app @@ -75,7 +75,6 @@ def test_should_send_personalised_template_to_correct_sms_provider_and_persist( reply_to_text=sample_sms_template_with_html.service.get_default_sms_sender()) mocker.patch('app.mmg_client.send_sms') - stats_mock = mocker.patch('app.delivery.send_to_providers.create_initial_notification_statistic_tasks') send_to_providers.send_sms_to_provider( db_notification @@ -88,8 +87,6 @@ def test_should_send_personalised_template_to_correct_sms_provider_and_persist( sender=current_app.config['FROM_NUMBER'] ) - stats_mock.assert_called_once_with(db_notification) - notification = Notification.query.filter_by(id=db_notification.id).one() assert notification.status == 'sending' @@ -110,7 +107,6 @@ def test_should_send_personalised_template_to_correct_email_provider_and_persist ) mocker.patch('app.aws_ses_client.send_email', return_value='reference') - stats_mock = mocker.patch('app.delivery.send_to_providers.create_initial_notification_statistic_tasks') send_to_providers.send_email_to_provider( db_notification @@ -124,7 +120,6 @@ def test_should_send_personalised_template_to_correct_email_provider_and_persist html_body=ANY, reply_to_address=None ) - stats_mock.assert_called_once_with(db_notification) assert '