Move notification retrieval logic into perf platform client and dont filter by status

app/celery/scheduled_tasks.py

@@ -1,4 +1,4 @@
-from datetime import date, datetime, timedelta
+from datetime import datetime
 
 from flask import current_app
 from sqlalchemy.exc import SQLAlchemyError
@@ -10,8 +10,7 @@ from app.dao.invited_user_dao import delete_invitations_created_more_than_two_da
 from app.dao.jobs_dao import dao_set_scheduled_jobs_to_pending, dao_get_jobs_older_than
 from app.dao.notifications_dao import (
     delete_notifications_created_more_than_a_week_ago,
-    dao_timeout_notifications,
+    dao_timeout_notifications
-    get_total_sent_notifications_yesterday
 )
 from app.dao.users_dao import delete_codes_older_created_more_than_a_day_ago
 from app.statsd_decorators import statsd
@@ -118,7 +117,7 @@ def timeout_notifications():
 @notify_celery.task(name='send-daily-performance-platform-stats')
 @statsd(namespace="tasks")
 def send_daily_performance_stats():
-    count_dict = get_total_sent_notifications_yesterday()
+    count_dict = performance_platform_client.get_total_sent_notifications_yesterday()
     start_date = count_dict.get('start_date')
 
     performance_platform_client.send_performance_stats(

app/clients/performance_platform/performance_platform_client.py

@@ -1,9 +1,15 @@
 import base64
 import json
+from datetime import datetime
 from requests import request
 
 from flask import current_app
 
+from app.utils import (
+    get_midnight_for_day_before,
+    get_london_midnight_in_utc
+)
+
 
 class PerformancePlatformClient:
 
@@ -26,6 +32,23 @@ class PerformancePlatformClient:
             self._add_id_for_payload(payload)
             self._send_stats_to_performance_platform(payload)
 
+    def get_total_sent_notifications_yesterday(self):
+        today = datetime.utcnow()
+        start_date = get_midnight_for_day_before(today)
+        end_date = get_london_midnight_in_utc(today)
+
+        from app.dao.notifications_dao import get_total_sent_notifications_in_date_range
+        return {
+            "start_date": start_date,
+            "end_date": end_date,
+            "email": {
+                "count": get_total_sent_notifications_in_date_range(start_date, end_date, 'email')
+            },
+            "sms": {
+                "count": get_total_sent_notifications_in_date_range(start_date, end_date, 'sms')
+            }
+        }
+
     def _send_stats_to_performance_platform(self, payload):
         headers = {
             'Content-Type': "application/json",

app/dao/notifications_dao.py

@@ -22,15 +22,11 @@ from app.models import (
     NOTIFICATION_PENDING,
     NOTIFICATION_TECHNICAL_FAILURE,
     NOTIFICATION_TEMPORARY_FAILURE,
-    NOTIFICATION_STATUS_TYPES_COMPLETED,
     KEY_TYPE_NORMAL, KEY_TYPE_TEST
 )
 
 from app.dao.dao_utils import transactional
 from app.statsd_decorators import statsd
-from app.utils import (
-    get_midnight_for_day_before,
-    get_london_midnight_in_utc)
 
 
 def dao_get_notification_statistics_for_service_and_day(service_id, day):
@@ -419,27 +415,9 @@ def get_total_sent_notifications_in_date_range(start_date, end_date, notificatio
         func.count(NotificationHistory.id).label('count')
     ).filter(
         NotificationHistory.key_type != KEY_TYPE_TEST,
-        NotificationHistory.status.in_(NOTIFICATION_STATUS_TYPES_COMPLETED),
         NotificationHistory.created_at >= start_date,
         NotificationHistory.created_at <= end_date,
         NotificationHistory.notification_type == notification_type
-    ).first()
+    ).scalar()
 
-    return 0 if result is None else result.count
+    return result or 0
-
-
-def get_total_sent_notifications_yesterday():
-    today = datetime.utcnow()
-    start_date = get_midnight_for_day_before(today)
-    end_date = get_london_midnight_in_utc(today)
-
-    return {
-        "start_date": start_date,
-        "end_date": end_date,
-        "email": {
-            "count": get_total_sent_notifications_in_date_range(start_date, end_date, 'email')
-        },
-        "sms": {
-            "count": get_total_sent_notifications_in_date_range(start_date, end_date, 'sms')
-        }
-    }

app/utils.py

@@ -2,7 +2,6 @@ from datetime import datetime, timedelta
 
 import pytz
 from flask import url_for
-from app.models import SMS_TYPE, EMAIL_TYPE
 from notifications_utils.template import SMSMessageTemplate, PlainTextEmailTemplate
 
 
@@ -26,6 +25,7 @@ def url_with_token(data, url, config):
 
 
 def get_template_instance(template, values):
+    from app.models import SMS_TYPE, EMAIL_TYPE
     return {
         SMS_TYPE: SMSMessageTemplate, EMAIL_TYPE: PlainTextEmailTemplate
     }[template['template_type']](template, values)

tests/app/clients/test_performance_platform.py

@@ -1,7 +1,15 @@
 import requests_mock
 import pytest
+from datetime import datetime
+from freezegun import freeze_time
+from functools import partial
 
 from app.clients.performance_platform.performance_platform_client import PerformancePlatformClient
+from app.utils import (
+    get_london_midnight_in_utc,
+    get_midnight_for_day_before
+)
+from tests.app.conftest import sample_notification_history as create_notification_history
 
 
 @pytest.fixture(scope='function')
@@ -69,3 +77,44 @@ def test_send_platform_stats_creates_correct_call(notify_api, client):
     assert request_args['count'] == 142
     expected_base64_id = 'MjAxNi0xMC0xNlQwMDowMDowMCswMDowMGdvdnVrLW5vdGlmeXNtc25vdGlmaWNhdGlvbnNkYXk='
     assert request_args['_id'] == expected_base64_id
+
+
+@freeze_time("2016-01-11 12:30:00")
+def test_get_total_sent_notifications_yesterday_returns_expected_totals_dict(
+    notify_db,
+    notify_db_session,
+    client,
+    sample_template
+):
+    notification_history = partial(
+        create_notification_history,
+        notify_db,
+        notify_db_session,
+        sample_template,
+        status='delivered'
+    )
+
+    notification_history(notification_type='email')
+    notification_history(notification_type='sms')
+
+    # Create some notifications for the day before
+    yesterday = datetime(2016, 1, 10, 15, 30, 0, 0)
+    with freeze_time(yesterday):
+        notification_history(notification_type='sms')
+        notification_history(notification_type='sms')
+        notification_history(notification_type='email')
+        notification_history(notification_type='email')
+        notification_history(notification_type='email')
+
+    total_count_dict = client.get_total_sent_notifications_yesterday()
+
+    assert total_count_dict == {
+        "start_date": get_midnight_for_day_before(datetime.utcnow()),
+        "end_date": get_london_midnight_in_utc(datetime.utcnow()),
+        "email": {
+            "count": 3
+        },
+        "sms": {
+            "count": 2
+        }
+    }

tests/app/dao/test_notification_dao.py

@@ -35,7 +35,6 @@ from app.dao.notifications_dao import (
     get_notifications_for_job,
     get_notifications_for_service,
     get_total_sent_notifications_in_date_range,
-    get_total_sent_notifications_yesterday,
     update_notification_status_by_id,
     update_notification_status_by_reference,
     dao_delete_notifications_and_history_by_id,
@@ -44,12 +43,6 @@ from app.dao.notifications_dao import (
     get_april_fools)
 
 from app.dao.services_dao import dao_update_service
-
-from app.utils import (
-    get_london_midnight_in_utc,
-    get_midnight_for_day_before
-)
-
 from tests.app.conftest import (
     sample_notification,
     sample_template,
@@ -1351,33 +1344,6 @@ def test_get_total_sent_notifications_in_date_range_returns_only_in_date_range(
     assert total_count == 3
 
 
-@pytest.mark.parametrize('notification_type', ['sms', 'email'])
-def test_get_total_sent_notifications_in_date_range_excludes_created_and_sending(
-    notify_db,
-    notify_db_session,
-    sample_template,
-    notification_type
-):
-    notification_history = partial(
-        create_notification_history,
-        notify_db,
-        notify_db_session,
-        sample_template,
-        notification_type=notification_type
-    )
-
-    start_date = datetime(2000, 3, 30, 0, 0, 0, 0)
-    end_date = datetime(2000, 3, 31, 0, 0, 0, 0)
-    with freeze_time(start_date):
-        notification_history(status='sending')
-        notification_history(status='created')
-        notification_history(status='failed')
-        notification_history(status='delivered')
-
-        total_count = get_total_sent_notifications_in_date_range(start_date, end_date, notification_type)
-        assert total_count == 2
-
-
 @pytest.mark.parametrize('notification_type', ['sms', 'email'])
 def test_get_total_sent_notifications_in_date_range_excludes_test_key_notifications(
     notify_db,
@@ -1432,7 +1398,7 @@ def test_get_total_sent_notifications_for_sms_excludes_email_counts(
     assert total_count == 3
 
 
-def test_get_total_sent_notifications_for_sms_excludes_sms_counts(
+def test_get_total_sent_notifications_for_email_excludes_sms_counts(
     notify_db,
     notify_db_session,
     sample_template
@@ -1456,43 +1422,3 @@ def test_get_total_sent_notifications_for_sms_excludes_sms_counts(
 
     total_count = get_total_sent_notifications_in_date_range(start_date, end_date, 'email')
     assert total_count == 2
-
-
-@freeze_time("2016-01-11 12:30:00")
-def test_get_total_sent_notifications_yesterday_returns_expected_totals_dict(
-    notify_db,
-    notify_db_session,
-    sample_template
-):
-    notification_history = partial(
-        create_notification_history,
-        notify_db,
-        notify_db_session,
-        sample_template,
-        status='delivered'
-    )
-
-    notification_history(notification_type='email')
-    notification_history(notification_type='sms')
-
-    # Create some notifications for the day before
-    yesterday = datetime(2016, 1, 10, 15, 30, 0, 0)
-    with freeze_time(yesterday):
-        notification_history(notification_type='sms')
-        notification_history(notification_type='sms')
-        notification_history(notification_type='email')
-        notification_history(notification_type='email')
-        notification_history(notification_type='email')
-
-    total_count_dict = get_total_sent_notifications_yesterday()
-
-    assert total_count_dict == {
-        "start_date": get_midnight_for_day_before(datetime.utcnow()),
-        "end_date": get_london_midnight_in_utc(datetime.utcnow()),
-        "email": {
-            "count": 3
-        },
-        "sms": {
-            "count": 2
-        }
-    }