Check for services sending sms messages to tv numbers

app/celery/scheduled_tasks.py

@@ -35,6 +35,7 @@ from app.dao.notifications_dao import (
 )
 from app.dao.provider_details_dao import dao_reduce_sms_provider_priority
 from app.dao.users_dao import delete_codes_older_created_more_than_a_day_ago
+from app.dao.services_dao import dao_find_services_sending_to_tv_numbers
 from app.models import (
     Job,
     JOB_STATUS_IN_PROGRESS,
@@ -260,16 +261,33 @@ def check_for_missing_rows_in_completed_jobs():
 @notify_celery.task(name='check-for-services-with-high-failure-rates-or-sending-to-tv-numbers')
 @statsd(namespace="tasks")
 def check_for_services_with_high_failure_rates_or_sending_to_tv_numbers():
-    services_with_failures = get_services_with_high_failure_rates()
+    start_date = (datetime.utcnow() - timedelta(days=1))
-    # services_sending_to_tv_numbers = dao_find_services_sending_to_tv_numbers(number=100)
+    end_date = datetime.utcnow()
+    message = ""
+
+    services_with_failures = get_services_with_high_failure_rates(start_date=start_date, end_date=end_date)
+    services_sending_to_tv_numbers = dao_find_services_sending_to_tv_numbers(
+        threshold=100,
+        start_date=start_date,
+        end_date=end_date
+    )
 
     if services_with_failures:
-        message = "{} service(s) have had high permanent-failure rates for sms messages in last 24 hours: ".format(
+        message += "{} service(s) have had high permanent-failure rates for sms messages in last 24 hours:\n".format(
             len(services_with_failures)
         )
         for service in services_with_failures:
-            message += "service id: {} failure rate: {}, ".format(service["id"], service["permanent_failure_rate"])
+            message += "service id: {} failure rate: {},\n".format(service["id"], service["permanent_failure_rate"])
+    elif services_sending_to_tv_numbers:
+        message += "{} service(s) have sent over 100 sms messages to tv numbers in last 24 hours:\n".format(
+            len(services_sending_to_tv_numbers)
+        )
+        for service in services_sending_to_tv_numbers:
+            message += "service id: {}, count of sms to tv numbers: {},\n".format(
+                service.service_id, service.notification_count
+            )
 
+    if services_with_failures or services_sending_to_tv_numbers:
         current_app.logger.exception(message)
 
         if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']:

app/dao/services_dao.py

@@ -521,3 +521,27 @@ def dao_fetch_active_users_for_service(service_id):
     )
 
     return query.all()
+
+
+def dao_find_services_sending_to_tv_numbers(start_date, end_date, threshold=100):
+
+    return db.session.query(
+        Service.name.label('service_name'),
+        Notification.service_id.label('service_id'),
+        func.count(Notification.id).label('notification_count')
+    ).filter(
+        Notification.service_id == Service.id,
+        Notification.created_at >= start_date,
+        Notification.created_at <= end_date,
+        Notification.key_type != KEY_TYPE_TEST,
+        Notification.notification_type == SMS_TYPE,
+        func.substr(Notification.normalised_to, 3, 7) == '7700900',
+        Service.restricted == False,  # noqa
+        Service.research_mode == False,
+        Service.active == True,
+    ).group_by(
+        Notification.service_id,
+        Service.name
+    ).having(
+        func.count(Notification.id) > threshold
+    ).all()

app/service/utils.py

@@ -8,7 +8,6 @@ from app.models import (
     KEY_TYPE_TEST, KEY_TYPE_TEAM, KEY_TYPE_NORMAL)
 
 from app.service import statistics
-from datetime import datetime, timedelta
 
 from app.dao.fact_notification_status_dao import fetch_stats_for_all_services_by_date_range
 
@@ -59,13 +58,10 @@ def service_allowed_to_send_to(recipient, service, key_type, allow_whitelisted_r
         )
 
 
-def get_services_with_high_failure_rates(rate=0.25, threshold=100):
+def get_services_with_high_failure_rates(start_date, end_date, rate=0.25, threshold=100):
-    start_date = (datetime.utcnow() - timedelta(days=1)).date()
-    end_date = datetime.utcnow().date()
-
     stats = fetch_stats_for_all_services_by_date_range(
-        start_date=start_date,
+        start_date=start_date.date(),
-        end_date=end_date,
+        end_date=end_date.date(),
         include_from_test_key=False,
     )
     results = []

tests/app/celery/test_scheduled_tasks.py

@@ -2,6 +2,7 @@ from datetime import datetime, timedelta
 from unittest.mock import call
 
 import pytest
+from collections import namedtuple
 from freezegun import freeze_time
 from mock import mock
 
@@ -497,25 +498,46 @@ def test_check_for_missing_rows_in_completed_jobs_uses_sender_id(mocker, sample_
     )
 
 
-@pytest.mark.parametrize("failure_rates, expected_message", [
+MockServicesSendingToTVNumbers = namedtuple(
+    'ServicesSendingToTVNumbers',
+    [
+        'service_id',
+        'service_name',
+        'notification_count',
+    ]
+)
+
+
+@pytest.mark.parametrize("failure_rates, sms_to_tv_numbers, expected_message", [
     [
         [{"id": "123", "name": "Service 1", "permanent_failure_rate": 0.3}],
+        [],
         "1 service(s) have had high permanent-failure rates for sms messages in last "
-        "24 hours: service id: 123 failure rate: 0.3, "
+        "24 hours:\nservice id: 123 failure rate: 0.3,\n"
+    ],
+    [
+        [],
+        [MockServicesSendingToTVNumbers("123", "Service 1", 300)],
+        "1 service(s) have sent over 100 sms messages to tv numbers in last 24 hours:\n"
+        "service id: 123, count of sms to tv numbers: 300,\n"
     ]
 ])
 def test_check_for_services_with_high_failure_rates_or_sending_to_tv_numbers(
-    mocker, notify_db_session, failure_rates, expected_message
+    mocker, notify_db_session, failure_rates, sms_to_tv_numbers, expected_message
 ):
     mock_logger = mocker.patch('app.celery.tasks.current_app.logger.exception')
     mock_create_ticket = mocker.patch('app.celery.scheduled_tasks.zendesk_client.create_ticket')
     mock_failure_rates = mocker.patch(
         'app.celery.scheduled_tasks.get_services_with_high_failure_rates', return_value=failure_rates
     )
+    mock_sms_to_tv_numbers = mocker.patch(
+        'app.celery.scheduled_tasks.dao_find_services_sending_to_tv_numbers', return_value=sms_to_tv_numbers
+    )
 
     check_for_services_with_high_failure_rates_or_sending_to_tv_numbers()
 
     assert mock_failure_rates.called
+    assert mock_sms_to_tv_numbers.called
     mock_logger.assert_called_once_with(expected_message)
     mock_create_ticket.assert_called_with(
         message=expected_message,

tests/app/dao/test_fact_notification_status_dao.py

@@ -14,7 +14,7 @@ from app.dao.fact_notification_status_dao import (
     fetch_notification_status_totals_for_all_services,
     fetch_notification_statuses_for_job,
     fetch_stats_for_all_services_by_date_range, fetch_monthly_template_usage_for_service,
-    get_total_sent_notifications_for_day_and_type
+    get_total_sent_notifications_for_day_and_type,
 )
 from app.models import (
     FactNotificationStatus,

tests/app/dao/test_services_dao.py

@@ -1,5 +1,5 @@
 import uuid
-from datetime import datetime
+from datetime import datetime, timedelta
 from unittest import mock
 
 import pytest
@@ -23,6 +23,7 @@ from app.dao.services_dao import (
     dao_fetch_live_services_data,
     dao_fetch_service_by_id,
     dao_fetch_all_services_by_user,
+    dao_find_services_sending_to_tv_numbers,
     dao_update_service,
     delete_service_and_all_associated_db_objects,
     dao_fetch_stats_for_service,
@@ -1099,3 +1100,41 @@ def create_email_sms_letter_template():
     template_two = create_template(service=service, template_name='2', template_type='sms')
     template_three = create_template(service=service, template_name='3', template_type='letter')
     return template_one, template_three, template_two
+
+
+@freeze_time("2019-12-02 12:00:00.000000")
+def test_dao_find_services_sending_to_tv_numbers(notify_db_session):
+    service_1 = create_service(service_name="Service 1")
+    service_3 = create_service(service_name="Service 3", restricted=True)  # restricted
+    service_4 = create_service(service_name="Service 4", research_mode=True)  # research mode
+    service_5 = create_service(service_name="Service 5", active=False)  # not active
+    services = [service_1, service_3, service_4, service_5]
+
+    tv_number = "447700900001"
+    normal_number = "447711900001"
+    normal_number_resembling_tv_number = "447227700900"
+
+    for service in services:
+        template = create_template(service)
+        for x in range(0, 5):
+            create_notification(template, normalised_to=tv_number, status="permanent-failure")
+
+    service_6 = create_service(service_name="Service 6")  # notifications too old
+    with freeze_time("2019-11-30 15:00:00.000000"):
+        template_6 = create_template(service_6)
+        for x in range(0, 5):
+            create_notification(template_6, normalised_to=tv_number, status="permanent-failure")
+
+    service_2 = create_service(service_name="Service 2")  # below threshold
+    template_2 = create_template(service_2)
+    create_notification(template_2, normalised_to=tv_number, status="permanent-failure")
+    for x in range(0, 5):
+        create_notification(template_2, normalised_to=normal_number, status="delivered")
+        create_notification(template_2, normalised_to=normal_number_resembling_tv_number, status="delivered")
+
+    start_date = (datetime.utcnow() - timedelta(days=1))
+    end_date = datetime.utcnow()
+
+    result = dao_find_services_sending_to_tv_numbers(start_date, end_date, threshold=4)
+    assert len(result) == 1
+    assert result[0].service_name == "Service 1"

tests/app/service/test_utils.py

@@ -2,6 +2,7 @@ from app.dao.date_util import get_current_financial_year_start_year
 from freezegun import freeze_time
 from tests.app.db import create_service, create_notification, create_template
 from app.service.utils import get_services_with_high_failure_rates
+from datetime import datetime, timedelta
 
 
 # see get_financial_year for conversion of financial years.
@@ -39,8 +40,10 @@ def test_get_services_with_high_failure_rates(notify_db_session):
     service_2 = create_service(service_name="Service 2")  # below threshold
     template_2 = create_template(service_2)
     create_notification(template_2, status="permanent-failure")
+    start_date = (datetime.utcnow() - timedelta(days=1))
+    end_date = datetime.utcnow()
 
-    assert get_services_with_high_failure_rates(threshold=3) == [{
+    assert get_services_with_high_failure_rates(start_date, end_date, threshold=3) == [{
         'id': str(service_1.id),
         'name': service_1.name,
         'permanent_failure_rate': 0.25