cleanup, get rid of print statements, etc.

app/celery/provider_tasks.py

@@ -44,7 +44,7 @@ def check_sms_delivery_receipt(self, message_id, notification_id, sent_at):
 
     if status == NOTIFICATION_DELIVERED:
         sanitize_notifications_by_id(notification_id)
-        current_app.logger.info(f"Did not Archive notification {notification_id} that was successfully delivered")
+        current_app.logger.info(f"Sanitized notification {notification_id} that was successfully delivered")
 
 
 @notify_celery.task(bind=True, name="deliver_sms", max_retries=48, default_retry_delay=300)

app/dao/fact_notification_status_dao.py

@@ -23,6 +23,7 @@ from app.models import (
     NOTIFICATION_TEMPORARY_FAILURE,
     FactNotificationStatus,
     Notification,
+    NotificationAllTimeView,
     Service,
     Template,
 )
@@ -37,7 +38,6 @@ from app.utils import (
 def update_fact_notification_status(process_day, notification_type, service_id):
     start_date = get_midnight_in_utc(process_day)
     end_date = get_midnight_in_utc(process_day + timedelta(days=1))
-    print(f"fact start {start_date} and end {end_date}")
 
     # delete any existing rows in case some no longer exist e.g. if all messages are sent
     FactNotificationStatus.query.filter(
@@ -48,25 +48,25 @@ def update_fact_notification_status(process_day, notification_type, service_id):
 
     query = db.session.query(
         literal(process_day).label("process_day"),
-        Notification.template_id,
+        NotificationAllTimeView.template_id,
         literal(service_id).label("service_id"),
-        func.coalesce(Notification.job_id, '00000000-0000-0000-0000-000000000000').label('job_id'),
+        func.coalesce(NotificationAllTimeView.job_id, '00000000-0000-0000-0000-000000000000').label('job_id'),
         literal(notification_type).label("notification_type"),
-        Notification.key_type,
+        NotificationAllTimeView.key_type,
-        Notification.status,
+        NotificationAllTimeView.status,
         func.count().label('notification_count')
     ).filter(
-        Notification.created_at >= start_date,
+        NotificationAllTimeView.created_at >= start_date,
-        Notification.created_at < end_date,
+        NotificationAllTimeView.created_at < end_date,
-        Notification.notification_type == notification_type,
+        NotificationAllTimeView.notification_type == notification_type,
-        Notification.service_id == service_id,
+        NotificationAllTimeView.service_id == service_id,
-        Notification.key_type.in_((KEY_TYPE_NORMAL, KEY_TYPE_TEAM)),
+        NotificationAllTimeView.key_type.in_((KEY_TYPE_NORMAL, KEY_TYPE_TEAM)),
     ).group_by(
-        Notification.template_id,
+        NotificationAllTimeView.template_id,
-        Notification.template_id,
+        NotificationAllTimeView.template_id,
         'job_id',
-        Notification.key_type,
+        NotificationAllTimeView.key_type,
-        Notification.status
+        NotificationAllTimeView.status
     )
 
     db.session.connection().execute(
@@ -167,14 +167,12 @@ def fetch_notification_status_for_service_for_today_and_7_previous_days(service_
     if by_template:
         query = query.filter(all_stats_table.c.template_id == Template.id)
 
-    x = query.group_by(
+    return query.group_by(
         *([Template.name, all_stats_table.c.template_id] if by_template else []),
         all_stats_table.c.notification_type,
         all_stats_table.c.status,
     ).all()
 
-    return x
-
 
 def fetch_notification_status_totals_for_all_services(start_date, end_date):
     stats = db.session.query(

app/dao/notifications_dao.py

@@ -238,7 +238,6 @@ def get_notifications_for_service(
 
     query = Notification.query.filter(*filters)
     query = _filter_query(query, filter_dict)
-    print(f"QUERY IS {query}")
     if personalisation:
         query = query.options(
             joinedload('template')
@@ -262,16 +261,10 @@ def _filter_query(query, filter_dict=None):
     statuses = multidict.getlist('status')
 
     if statuses:
-        statuses = Notification.substitute_status(statuses)
+        # statuses = Notification.substitute_status(statuses)
-        # TODO WHY
-        if len(statuses) == 5 and 'temporary-failure' in statuses:
-            statuses = ['failed']
-        elif len(statuses) == 10:
-            statuses = ['failed', 'sending', 'delivered']
-        elif statuses == ['pending']:
-            statuses = ['sending']
-        print(f"STATUSES = {statuses}")
 
+        if statuses == ['pending']:
+            statuses = ['sending']
         query = query.filter(Notification.status.in_(statuses))
 
     # filter by template

app/service/rest.py

@@ -354,7 +354,6 @@ def get_service_history(service_id):
 def get_all_notifications_for_service(service_id):
     if request.method == 'GET':
         data = notifications_filter_schema.load(request.args)
-        print(f"DATA IS {data}")
     elif request.method == 'POST':
         # Must transform request.get_json() to MultiDict as NotificationsFilterSchema expects a MultiDict.
         # Unlike request.args, request.get_json() does not return a MultiDict but instead just a dict.
@@ -388,7 +387,6 @@ def get_all_notifications_for_service(service_id):
         include_from_test_key=include_from_test_key,
         include_one_off=include_one_off
     )
-    print(f"PAGINATION ITEMS =  {pagination.items}")
 
     kwargs = request.args.to_dict()
     kwargs['service_id'] = service_id
@@ -397,7 +395,6 @@ def get_all_notifications_for_service(service_id):
         notifications = [notification.serialize_for_csv() for notification in pagination.items]
     else:
         notifications = notification_with_template_schema.dump(pagination.items, many=True)
-    print(f"NOTIFICATIONS AFTER FORMATTING {notifications}")
     # We try and get the next page of results to work out if we need provide a pagination link to the next page
     # in our response if it exists. Note, this could be done instead by changing `count_pages` in the previous
     # call to be True which will enable us to use Flask-Sqlalchemy to tell if there is a next page of results but
@@ -516,13 +513,10 @@ def get_detailed_service(service_id, today_only=False):
 def get_service_statistics(service_id, today_only, limit_days=7):
     # today_only flag is used by the send page to work out if the service will exceed their daily usage by sending a job
     if today_only:
-        print("TODAY ONLY")
         stats = dao_fetch_todays_stats_for_service(service_id)
     else:
-        print("PAST WEEK")
         stats = fetch_notification_status_for_service_for_today_and_7_previous_days(service_id, limit_days=limit_days)
 
-    print(f"GET_SERVICE_STATISTICS returns {statistics.format_statistics(stats)}")
     return statistics.format_statistics(stats)
 
 

tests/app/celery/test_reporting_tasks.py

@@ -1,5 +1,6 @@
 from datetime import date, datetime, time, timedelta
 from decimal import Decimal
+from uuid import UUID
 
 import pytest
 from freezegun import freeze_time
@@ -428,7 +429,7 @@ def test_create_nightly_notification_status_for_service_and_day(notify_db_sessio
     process_day = datetime.utcnow().date() - timedelta(days=5)
     with freeze_time(datetime.combine(process_day, time.max)):
         create_notification(template=first_template, status='delivered')
-        create_notification(template=second_template, status='delivered')
+        create_notification(template=second_template, status='failed')
 
         # team API key notifications are included
         create_notification(template=second_template, status='pending', key_type=KEY_TYPE_TEAM)
@@ -454,7 +455,7 @@ def test_create_nightly_notification_status_for_service_and_day(notify_db_sessio
         FactNotificationStatus.notification_status,
     ).all()
 
-    assert len(new_fact_data) == 3
+    assert len(new_fact_data) == 4
 
     email_delivered_row = new_fact_data[0]
     assert email_delivered_row.template_id == second_template.id
@@ -468,11 +469,21 @@ def test_create_nightly_notification_status_for_service_and_day(notify_db_sessio
     assert email_sending_row.template_id == second_template.id
     assert email_sending_row.service_id == second_service.id
     assert email_sending_row.notification_type == 'email'
-    assert email_sending_row.notification_status == 'pending'
+    assert email_sending_row.notification_status == 'failed'
     assert email_sending_row.notification_count == 1
-    assert email_sending_row.key_type == KEY_TYPE_TEAM
+    assert email_sending_row.key_type == KEY_TYPE_NORMAL
 
-    sms_delivered_row = new_fact_data[2]
+    email_failure_row = new_fact_data[2]
+    assert email_failure_row.local_date == process_day
+    assert email_failure_row.template_id == second_template.id
+    assert email_failure_row.service_id == second_service.id
+    assert email_failure_row.job_id == UUID('00000000-0000-0000-0000-000000000000')
+    assert email_failure_row.notification_type == 'email'
+    assert email_failure_row.notification_status == 'pending'
+    assert email_failure_row.notification_count == 1
+    assert email_failure_row.key_type == KEY_TYPE_TEAM
+
+    sms_delivered_row = new_fact_data[3]
     assert sms_delivered_row.template_id == first_template.id
     assert sms_delivered_row.service_id == first_service.id
     assert sms_delivered_row.notification_type == 'sms'