diff --git a/app/celery/provider_tasks.py b/app/celery/provider_tasks.py index cfc5438a0..3ae0486bc 100644 --- a/app/celery/provider_tasks.py +++ b/app/celery/provider_tasks.py @@ -44,7 +44,7 @@ def check_sms_delivery_receipt(self, message_id, notification_id, sent_at): if status == NOTIFICATION_DELIVERED: sanitize_notifications_by_id(notification_id) - current_app.logger.info(f"Did not Archive notification {notification_id} that was successfully delivered") + current_app.logger.info(f"Sanitized notification {notification_id} that was successfully delivered") @notify_celery.task(bind=True, name="deliver_sms", max_retries=48, default_retry_delay=300) diff --git a/app/dao/fact_notification_status_dao.py b/app/dao/fact_notification_status_dao.py index 6fce9624d..857256cd3 100644 --- a/app/dao/fact_notification_status_dao.py +++ b/app/dao/fact_notification_status_dao.py @@ -23,6 +23,7 @@ from app.models import ( NOTIFICATION_TEMPORARY_FAILURE, FactNotificationStatus, Notification, + NotificationAllTimeView, Service, Template, ) @@ -37,7 +38,6 @@ from app.utils import ( def update_fact_notification_status(process_day, notification_type, service_id): start_date = get_midnight_in_utc(process_day) end_date = get_midnight_in_utc(process_day + timedelta(days=1)) - print(f"fact start {start_date} and end {end_date}") # delete any existing rows in case some no longer exist e.g. if all messages are sent FactNotificationStatus.query.filter( @@ -48,25 +48,25 @@ def update_fact_notification_status(process_day, notification_type, service_id): query = db.session.query( literal(process_day).label("process_day"), - Notification.template_id, + NotificationAllTimeView.template_id, literal(service_id).label("service_id"), - func.coalesce(Notification.job_id, '00000000-0000-0000-0000-000000000000').label('job_id'), + func.coalesce(NotificationAllTimeView.job_id, '00000000-0000-0000-0000-000000000000').label('job_id'), literal(notification_type).label("notification_type"), - Notification.key_type, - Notification.status, + NotificationAllTimeView.key_type, + NotificationAllTimeView.status, func.count().label('notification_count') ).filter( - Notification.created_at >= start_date, - Notification.created_at < end_date, - Notification.notification_type == notification_type, - Notification.service_id == service_id, - Notification.key_type.in_((KEY_TYPE_NORMAL, KEY_TYPE_TEAM)), + NotificationAllTimeView.created_at >= start_date, + NotificationAllTimeView.created_at < end_date, + NotificationAllTimeView.notification_type == notification_type, + NotificationAllTimeView.service_id == service_id, + NotificationAllTimeView.key_type.in_((KEY_TYPE_NORMAL, KEY_TYPE_TEAM)), ).group_by( - Notification.template_id, - Notification.template_id, + NotificationAllTimeView.template_id, + NotificationAllTimeView.template_id, 'job_id', - Notification.key_type, - Notification.status + NotificationAllTimeView.key_type, + NotificationAllTimeView.status ) db.session.connection().execute( @@ -167,14 +167,12 @@ def fetch_notification_status_for_service_for_today_and_7_previous_days(service_ if by_template: query = query.filter(all_stats_table.c.template_id == Template.id) - x = query.group_by( + return query.group_by( *([Template.name, all_stats_table.c.template_id] if by_template else []), all_stats_table.c.notification_type, all_stats_table.c.status, ).all() - return x - def fetch_notification_status_totals_for_all_services(start_date, end_date): stats = db.session.query( diff --git a/app/dao/notifications_dao.py b/app/dao/notifications_dao.py index 373c4cd65..435989563 100644 --- a/app/dao/notifications_dao.py +++ b/app/dao/notifications_dao.py @@ -238,7 +238,6 @@ def get_notifications_for_service( query = Notification.query.filter(*filters) query = _filter_query(query, filter_dict) - print(f"QUERY IS {query}") if personalisation: query = query.options( joinedload('template') @@ -262,16 +261,10 @@ def _filter_query(query, filter_dict=None): statuses = multidict.getlist('status') if statuses: - statuses = Notification.substitute_status(statuses) - # TODO WHY - if len(statuses) == 5 and 'temporary-failure' in statuses: - statuses = ['failed'] - elif len(statuses) == 10: - statuses = ['failed', 'sending', 'delivered'] - elif statuses == ['pending']: - statuses = ['sending'] - print(f"STATUSES = {statuses}") + # statuses = Notification.substitute_status(statuses) + if statuses == ['pending']: + statuses = ['sending'] query = query.filter(Notification.status.in_(statuses)) # filter by template diff --git a/app/service/rest.py b/app/service/rest.py index df5b328c3..06ee18a1d 100644 --- a/app/service/rest.py +++ b/app/service/rest.py @@ -354,7 +354,6 @@ def get_service_history(service_id): def get_all_notifications_for_service(service_id): if request.method == 'GET': data = notifications_filter_schema.load(request.args) - print(f"DATA IS {data}") elif request.method == 'POST': # Must transform request.get_json() to MultiDict as NotificationsFilterSchema expects a MultiDict. # Unlike request.args, request.get_json() does not return a MultiDict but instead just a dict. @@ -388,7 +387,6 @@ def get_all_notifications_for_service(service_id): include_from_test_key=include_from_test_key, include_one_off=include_one_off ) - print(f"PAGINATION ITEMS = {pagination.items}") kwargs = request.args.to_dict() kwargs['service_id'] = service_id @@ -397,7 +395,6 @@ def get_all_notifications_for_service(service_id): notifications = [notification.serialize_for_csv() for notification in pagination.items] else: notifications = notification_with_template_schema.dump(pagination.items, many=True) - print(f"NOTIFICATIONS AFTER FORMATTING {notifications}") # We try and get the next page of results to work out if we need provide a pagination link to the next page # in our response if it exists. Note, this could be done instead by changing `count_pages` in the previous # call to be True which will enable us to use Flask-Sqlalchemy to tell if there is a next page of results but @@ -516,13 +513,10 @@ def get_detailed_service(service_id, today_only=False): def get_service_statistics(service_id, today_only, limit_days=7): # today_only flag is used by the send page to work out if the service will exceed their daily usage by sending a job if today_only: - print("TODAY ONLY") stats = dao_fetch_todays_stats_for_service(service_id) else: - print("PAST WEEK") stats = fetch_notification_status_for_service_for_today_and_7_previous_days(service_id, limit_days=limit_days) - print(f"GET_SERVICE_STATISTICS returns {statistics.format_statistics(stats)}") return statistics.format_statistics(stats) diff --git a/tests/app/celery/test_reporting_tasks.py b/tests/app/celery/test_reporting_tasks.py index c541cc1a0..9e891327c 100644 --- a/tests/app/celery/test_reporting_tasks.py +++ b/tests/app/celery/test_reporting_tasks.py @@ -1,5 +1,6 @@ from datetime import date, datetime, time, timedelta from decimal import Decimal +from uuid import UUID import pytest from freezegun import freeze_time @@ -428,7 +429,7 @@ def test_create_nightly_notification_status_for_service_and_day(notify_db_sessio process_day = datetime.utcnow().date() - timedelta(days=5) with freeze_time(datetime.combine(process_day, time.max)): create_notification(template=first_template, status='delivered') - create_notification(template=second_template, status='delivered') + create_notification(template=second_template, status='failed') # team API key notifications are included create_notification(template=second_template, status='pending', key_type=KEY_TYPE_TEAM) @@ -454,7 +455,7 @@ def test_create_nightly_notification_status_for_service_and_day(notify_db_sessio FactNotificationStatus.notification_status, ).all() - assert len(new_fact_data) == 3 + assert len(new_fact_data) == 4 email_delivered_row = new_fact_data[0] assert email_delivered_row.template_id == second_template.id @@ -468,11 +469,21 @@ def test_create_nightly_notification_status_for_service_and_day(notify_db_sessio assert email_sending_row.template_id == second_template.id assert email_sending_row.service_id == second_service.id assert email_sending_row.notification_type == 'email' - assert email_sending_row.notification_status == 'pending' + assert email_sending_row.notification_status == 'failed' assert email_sending_row.notification_count == 1 - assert email_sending_row.key_type == KEY_TYPE_TEAM + assert email_sending_row.key_type == KEY_TYPE_NORMAL - sms_delivered_row = new_fact_data[2] + email_failure_row = new_fact_data[2] + assert email_failure_row.local_date == process_day + assert email_failure_row.template_id == second_template.id + assert email_failure_row.service_id == second_service.id + assert email_failure_row.job_id == UUID('00000000-0000-0000-0000-000000000000') + assert email_failure_row.notification_type == 'email' + assert email_failure_row.notification_status == 'pending' + assert email_failure_row.notification_count == 1 + assert email_failure_row.key_type == KEY_TYPE_TEAM + + sms_delivered_row = new_fact_data[3] assert sms_delivered_row.template_id == first_template.id assert sms_delivered_row.service_id == first_service.id assert sms_delivered_row.notification_type == 'sms'