darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-08-26 01:03:41 -04:00
Code Issues Packages Projects Releases Wiki Activity

Query directly for services with high failure rate

Browse Source
This commit is contained in:
Pea Tyczynska
2019-12-05 16:07:06 +00:00
parent b8de67ae54
commit 1b7b26bf24
8 changed files with 116 additions and 204 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
app/celery/scheduled_tasks.py
View File

@@ -35,7 +35,7 @@ from app.dao.notifications_dao import (
) )
from app.dao.provider_details_dao import dao_reduce_sms_provider_priority from app.dao.provider_details_dao import dao_reduce_sms_provider_priority
from app.dao.users_dao import delete_codes_older_created_more_than_a_day_ago from app.dao.users_dao import delete_codes_older_created_more_than_a_day_ago
from app.dao.services_dao import dao_find_services_sending_to_tv_numbers from app.dao.services_dao import dao_find_services_sending_to_tv_numbers, dao_find_services_with_high_failure_rates
from app.models import ( from app.models import (
Job, Job,
JOB_STATUS_IN_PROGRESS, JOB_STATUS_IN_PROGRESS,
@@ -46,8 +46,6 @@ from app.models import (
from app.notifications.process_notifications import send_notification_to_queue from app.notifications.process_notifications import send_notification_to_queue
from app.v2.errors import JobIncompleteError from app.v2.errors import JobIncompleteError
from app.service.utils import get_services_with_high_failure_rates
@notify_celery.task(name="run-scheduled-jobs") @notify_celery.task(name="run-scheduled-jobs")
@statsd(namespace="tasks") @statsd(namespace="tasks")
@@ -265,7 +263,7 @@ def check_for_services_with_high_failure_rates_or_sending_to_tv_numbers():
end_date = datetime.utcnow() end_date = datetime.utcnow()
message = "" message = ""
services_with_failures = get_services_with_high_failure_rates(start_date=start_date, end_date=end_date) services_with_failures = dao_find_services_with_high_failure_rates(start_date=start_date, end_date=end_date)
services_sending_to_tv_numbers = dao_find_services_sending_to_tv_numbers(start_date=start_date, end_date=end_date) services_sending_to_tv_numbers = dao_find_services_sending_to_tv_numbers(start_date=start_date, end_date=end_date)
if services_with_failures: if services_with_failures:
@@ -273,8 +271,8 @@ def check_for_services_with_high_failure_rates_or_sending_to_tv_numbers():
len(services_with_failures) len(services_with_failures)
) )
for service in services_with_failures: for service in services_with_failures:
service_dashboard = current_app.config['ADMIN_BASE_URL'] + "/services/" + service["id"] service_dashboard = current_app.config['ADMIN_BASE_URL'] + "/services/" + service.service_id
message += "service: {} failure rate: {},\n".format(service_dashboard, service["permanent_failure_rate"]) message += "service: {} failure rate: {},\n".format(service_dashboard, service.permanent_failure_rate)
elif services_sending_to_tv_numbers: elif services_sending_to_tv_numbers:
message += "{} service(s) have sent over 100 sms messages to tv numbers in last 24 hours:\n".format( message += "{} service(s) have sent over 100 sms messages to tv numbers in last 24 hours:\n".format(
len(services_sending_to_tv_numbers) len(services_sending_to_tv_numbers)

46
app/dao/services_dao.py
View File

@@ -4,6 +4,7 @@ from datetime import date, datetime, timedelta
from notifications_utils.statsd_decorators import statsd from notifications_utils.statsd_decorators import statsd
from sqlalchemy.sql.expression import asc, case, and_, func from sqlalchemy.sql.expression import asc, case, and_, func
from sqlalchemy.orm import joinedload from sqlalchemy.orm import joinedload
from sqlalchemy import cast, Float
from flask import current_app from flask import current_app
from app import db from app import db
@@ -44,6 +45,7 @@ from app.models import (
KEY_TYPE_TEST, KEY_TYPE_TEST,
NHS_ORGANISATION_TYPES, NHS_ORGANISATION_TYPES,
NON_CROWN_ORGANISATION_TYPES, NON_CROWN_ORGANISATION_TYPES,
NOTIFICATION_PERMANENT_FAILURE,
SMS_TYPE, SMS_TYPE,
LETTER_TYPE, LETTER_TYPE,
) )
@@ -544,12 +546,10 @@ def dao_find_services_sending_to_tv_numbers(start_date, end_date, threshold=100)
).all() ).all()
def dao_find_real_sms_notification_count_by_status_for_live_services(start_date, end_date): def dao_find_services_with_high_failure_rates(start_date, end_date, threshold=100):
# only works within services' retention period subquery = db.session.query(
return db.session.query( func.count(Notification.id).label('total_count'),
Notification.service_id.label('service_id'), Notification.service_id.label('service_id')
Notification.status.label('status'),
func.count(Notification.id).label('count')
).filter( ).filter(
Notification.service_id == Service.id, Notification.service_id == Service.id,
Notification.created_at >= start_date, Notification.created_at >= start_date,
@@ -561,5 +561,35 @@ def dao_find_real_sms_notification_count_by_status_for_live_services(start_date,
Service.active == True, Service.active == True,
).group_by( ).group_by(
Notification.service_id, Notification.service_id,
Notification.status ).having(
).all() func.count(Notification.id) >= threshold
)
subquery = subquery.subquery()
query = db.session.query(
Notification.service_id.label('service_id'),
func.count(Notification.id).label('permanent_failure_count'),
subquery.c.total_count.label('total_count'),
(cast(func.count(Notification.id), Float) / cast(subquery.c.total_count, Float)).label('permanent_failure_rate')
).join(
subquery,
subquery.c.service_id == Notification.service_id
).filter(
Notification.service_id == Service.id,
Notification.created_at >= start_date,
Notification.created_at <= end_date,
Notification.key_type != KEY_TYPE_TEST,
Notification.notification_type == SMS_TYPE,
Notification.status == NOTIFICATION_PERMANENT_FAILURE,
Service.restricted == False, # noqa
Service.research_mode == False,
Service.active == True,
).group_by(
Notification.service_id,
subquery.c.total_count
).having(
cast(func.count(Notification.id), Float) / cast(subquery.c.total_count, Float) >= 0.25
)
return query.all()

19
app/service/statistics.py
View File

@@ -21,18 +21,6 @@ def format_statistics(statistics):
return counts return counts
def get_rate_of_permanent_failures_for_service(statistics, threshold=100):
counts = {"permanent_failure": 0, "all_other_statuses": 0}
for row in statistics:
_count_if_status_is_permanent_failure_from_row(counts, row)
if counts['permanent_failure'] + counts['all_other_statuses'] >= threshold:
rate = counts['permanent_failure'] / (counts['permanent_failure'] + counts['all_other_statuses'])
else:
rate = 0
return rate
def format_admin_stats(statistics): def format_admin_stats(statistics):
counts = create_stats_dict() counts = create_stats_dict()
@@ -106,13 +94,6 @@ def _update_statuses_from_row(update_dict, row):
update_dict['failed'] += row.count update_dict['failed'] += row.count
def _count_if_status_is_permanent_failure_from_row(update_dict, row):
if row.status == 'permanent-failure':
update_dict['permanent_failure'] += row.count
else:
update_dict['all_other_statuses'] += row.count
def create_empty_monthly_notification_status_stats_dict(year): def create_empty_monthly_notification_status_stats_dict(year):
utc_month_starts = get_months_for_financial_year(year) utc_month_starts = get_months_for_financial_year(year)
# nested dicts - data[month][template type][status] = count # nested dicts - data[month][template type][status] = count

22
app/service/utils.py
View File

@@ -2,15 +2,11 @@ import itertools
from notifications_utils.recipients import allowed_to_send_to from notifications_utils.recipients import allowed_to_send_to
from app.dao.services_dao import dao_find_real_sms_notification_count_by_status_for_live_services
from app.models import ( from app.models import (
ServiceWhitelist, ServiceWhitelist,
MOBILE_TYPE, EMAIL_TYPE, MOBILE_TYPE, EMAIL_TYPE,
KEY_TYPE_TEST, KEY_TYPE_TEAM, KEY_TYPE_NORMAL) KEY_TYPE_TEST, KEY_TYPE_TEAM, KEY_TYPE_NORMAL)
from app.service import statistics
def get_recipients_from_request(request_json, key, type): def get_recipients_from_request(request_json, key, type):
return [(type, recipient) for recipient in request_json.get(key)] return [(type, recipient) for recipient in request_json.get(key)]
@@ -56,21 +52,3 @@ def service_allowed_to_send_to(recipient, service, key_type, allow_whitelisted_r
whitelist_members whitelist_members
) )
) )
def get_services_with_high_failure_rates(start_date, end_date, rate=0.25, threshold=100):
stats = dao_find_real_sms_notification_count_by_status_for_live_services(
start_date=start_date,
end_date=end_date,
)
results = []
for service_id, rows in itertools.groupby(stats, lambda x: x.service_id):
rows = list(rows)
permanent_failure_rate = statistics.get_rate_of_permanent_failures_for_service(rows, threshold=threshold)
if permanent_failure_rate >= rate:
results.append({
'id': str(rows[0].service_id),
'permanent_failure_rate': permanent_failure_rate
})
return results

11
tests/app/celery/test_scheduled_tasks.py
View File

@@ -505,11 +505,18 @@ MockServicesSendingToTVNumbers = namedtuple(
'notification_count', 'notification_count',
] ]
) )
MockServicesWithHighFailureRate = namedtuple(
'ServicesWithHighFailureRate',
[
'service_id',
'permanent_failure_rate',
]
)
@pytest.mark.parametrize("failure_rates, sms_to_tv_numbers, expected_message", [ @pytest.mark.parametrize("failure_rates, sms_to_tv_numbers, expected_message", [
[ [
[{"id": "123", "permanent_failure_rate": 0.3}], [MockServicesWithHighFailureRate("123", 0.3)],
[], [],
"1 service(s) have had high permanent-failure rates for sms messages in last " "1 service(s) have had high permanent-failure rates for sms messages in last "
"24 hours:\nservice: {} failure rate: 0.3,\n".format( "24 hours:\nservice: {} failure rate: 0.3,\n".format(
@@ -531,7 +538,7 @@ def test_check_for_services_with_high_failure_rates_or_sending_to_tv_numbers(
mock_logger = mocker.patch('app.celery.tasks.current_app.logger.exception') mock_logger = mocker.patch('app.celery.tasks.current_app.logger.exception')
mock_create_ticket = mocker.patch('app.celery.scheduled_tasks.zendesk_client.create_ticket') mock_create_ticket = mocker.patch('app.celery.scheduled_tasks.zendesk_client.create_ticket')
mock_failure_rates = mocker.patch( mock_failure_rates = mocker.patch(
'app.celery.scheduled_tasks.get_services_with_high_failure_rates', return_value=failure_rates 'app.celery.scheduled_tasks.dao_find_services_with_high_failure_rates', return_value=failure_rates
) )
mock_sms_to_tv_numbers = mocker.patch( mock_sms_to_tv_numbers = mocker.patch(
'app.celery.scheduled_tasks.dao_find_services_sending_to_tv_numbers', return_value=sms_to_tv_numbers 'app.celery.scheduled_tasks.dao_find_services_sending_to_tv_numbers', return_value=sms_to_tv_numbers

152
tests/app/dao/test_services_dao.py
View File

@@ -8,79 +8,49 @@ from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm.exc import NoResultFound from sqlalchemy.orm.exc import NoResultFound
from app import db from app import db
from app.dao.inbound_numbers_dao import ( from app.dao.inbound_numbers_dao import (dao_get_available_inbound_numbers,
dao_set_inbound_number_to_service, dao_set_inbound_number_active_flag,
dao_get_available_inbound_numbers, dao_set_inbound_number_to_service)
dao_set_inbound_number_active_flag
)
from app.dao.organisation_dao import dao_add_service_to_organisation from app.dao.organisation_dao import dao_add_service_to_organisation
from app.dao.service_permissions_dao import dao_add_service_permission, dao_remove_service_permission from app.dao.service_permissions_dao import (dao_add_service_permission,
from app.dao.services_dao import ( dao_remove_service_permission)
dao_create_service, from app.dao.service_user_dao import (dao_get_service_user,
dao_add_user_to_service, dao_update_service_user)
dao_remove_user_from_service, from app.dao.services_dao import (dao_add_user_to_service, dao_create_service,
dao_fetch_all_services, dao_fetch_active_users_for_service,
dao_fetch_live_services_data, dao_fetch_all_services,
dao_fetch_service_by_id, dao_fetch_all_services_by_user,
dao_fetch_all_services_by_user, dao_fetch_live_services_data,
dao_find_real_sms_notification_count_by_status_for_live_services, dao_fetch_service_by_id,
dao_find_services_sending_to_tv_numbers, dao_fetch_service_by_inbound_number,
dao_update_service, dao_fetch_stats_for_service,
delete_service_and_all_associated_db_objects, dao_fetch_todays_stats_for_all_services,
dao_fetch_stats_for_service, dao_fetch_todays_stats_for_service,
dao_fetch_todays_stats_for_service, dao_find_services_sending_to_tv_numbers,
fetch_todays_total_message_count, dao_find_services_with_high_failure_rates,
dao_fetch_todays_stats_for_all_services, dao_remove_user_from_service,
dao_suspend_service, dao_resume_service, dao_suspend_service,
dao_resume_service, dao_update_service,
dao_fetch_active_users_for_service, delete_service_and_all_associated_db_objects,
dao_fetch_service_by_inbound_number, fetch_todays_total_message_count,
get_services_by_partial_name, get_services_by_partial_name)
) from app.dao.users_dao import create_user_code, save_model_user
from app.dao.service_user_dao import dao_get_service_user, dao_update_service_user from app.models import (EMAIL_TYPE, INTERNATIONAL_SMS_TYPE, KEY_TYPE_NORMAL,
from app.dao.users_dao import save_model_user, create_user_code KEY_TYPE_TEAM, KEY_TYPE_TEST, LETTER_TYPE, SMS_TYPE,
from app.models import ( ApiKey, InvitedUser, Job, Notification,
VerifyCode, NotificationHistory, Organisation, Permission, Service,
ApiKey, ServicePermission, ServiceUser, Template,
Template, TemplateHistory, User, VerifyCode,
TemplateHistory, user_folder_permissions)
Job, from tests.app.db import (create_annual_billing, create_api_key,
Notification, create_email_branding, create_ft_billing,
NotificationHistory, create_inbound_number, create_invited_user,
Permission, create_letter_branding, create_notification,
User, create_notification_history, create_organisation,
InvitedUser, create_service,
Service, create_service_with_defined_sms_sender,
ServicePermission, create_service_with_inbound_number, create_template,
ServiceUser, create_template_folder, create_user)
KEY_TYPE_NORMAL,
KEY_TYPE_TEAM,
KEY_TYPE_TEST,
EMAIL_TYPE,
SMS_TYPE,
INTERNATIONAL_SMS_TYPE,
LETTER_TYPE,
user_folder_permissions,
Organisation
)
from tests.app.db import (
create_ft_billing,
create_inbound_number,
create_organisation,
create_user,
create_service,
create_service_with_inbound_number,
create_service_with_defined_sms_sender,
create_template,
create_template_folder,
create_notification,
create_api_key,
create_invited_user,
create_email_branding,
create_letter_branding,
create_notification_history,
create_annual_billing,
)
def test_should_have_decorated_services_dao_functions(): def test_should_have_decorated_services_dao_functions():
@@ -1106,9 +1076,9 @@ def create_email_sms_letter_template():
@freeze_time("2019-12-02 12:00:00.000000") @freeze_time("2019-12-02 12:00:00.000000")
def test_dao_find_services_sending_to_tv_numbers(notify_db_session, fake_uuid): def test_dao_find_services_sending_to_tv_numbers(notify_db_session, fake_uuid):
service_1 = create_service(service_name="Service 1", service_id=fake_uuid) service_1 = create_service(service_name="Service 1", service_id=fake_uuid)
service_3 = create_service(service_name="Service 3", restricted=True) # restricted service_3 = create_service(service_name="Service 3", restricted=True) # restricted is excluded
service_4 = create_service(service_name="Service 4", research_mode=True) # research mode service_4 = create_service(service_name="Service 4", research_mode=True) # research mode is excluded
service_5 = create_service(service_name="Service 5", active=False) # not active service_5 = create_service(service_name="Service 5", active=False) # not active is excluded
services = [service_1, service_3, service_4, service_5] services = [service_1, service_3, service_4, service_5]
tv_number = "447700900001" tv_number = "447700900001"
@@ -1120,13 +1090,13 @@ def test_dao_find_services_sending_to_tv_numbers(notify_db_session, fake_uuid):
for x in range(0, 5): for x in range(0, 5):
create_notification(template, normalised_to=tv_number, status="permanent-failure") create_notification(template, normalised_to=tv_number, status="permanent-failure")
service_6 = create_service(service_name="Service 6") # notifications too old service_6 = create_service(service_name="Service 6") # notifications too old are excluded
with freeze_time("2019-11-30 15:00:00.000000"): with freeze_time("2019-11-30 15:00:00.000000"):
template_6 = create_template(service_6) template_6 = create_template(service_6)
for x in range(0, 5): for x in range(0, 5):
create_notification(template_6, normalised_to=tv_number, status="permanent-failure") create_notification(template_6, normalised_to=tv_number, status="permanent-failure")
service_2 = create_service(service_name="Service 2") # below threshold service_2 = create_service(service_name="Service 2") # below threshold is excluded
template_2 = create_template(service_2) template_2 = create_template(service_2)
create_notification(template_2, normalised_to=tv_number, status="permanent-failure") create_notification(template_2, normalised_to=tv_number, status="permanent-failure")
for x in range(0, 5): for x in range(0, 5):
@@ -1141,11 +1111,11 @@ def test_dao_find_services_sending_to_tv_numbers(notify_db_session, fake_uuid):
assert str(result[0].service_id) == fake_uuid assert str(result[0].service_id) == fake_uuid
def test_dao_find_real_sms_notification_count_by_status_for_live_services(notify_db_session, fake_uuid): def test_dao_find_services_with_high_failure_rates(notify_db_session, fake_uuid):
service_1 = create_service(service_name="Service 1", service_id=fake_uuid) service_1 = create_service(service_name="Service 1", service_id=fake_uuid)
service_3 = create_service(service_name="Service 3", restricted=True) # restricted service_3 = create_service(service_name="Service 3", restricted=True) # restricted is excluded
service_4 = create_service(service_name="Service 4", research_mode=True) # research mode service_4 = create_service(service_name="Service 4", research_mode=True) # research mode is excluded
service_5 = create_service(service_name="Service 5", active=False) # not active service_5 = create_service(service_name="Service 5", active=False) # not active is excluded
services = [service_1, service_3, service_4, service_5] services = [service_1, service_3, service_4, service_5]
for service in services: for service in services:
@@ -1153,21 +1123,27 @@ def test_dao_find_real_sms_notification_count_by_status_for_live_services(notify
for x in range(0, 3): for x in range(0, 3):
create_notification(template, status="permanent-failure") create_notification(template, status="permanent-failure")
create_notification(template, status="delivered") create_notification(template, status="delivered")
create_notification(template, status="sending")
create_notification(template, status="temporary-failure") create_notification(template, status="temporary-failure")
service_6 = create_service(service_name="Service 6") # notifications too old service_6 = create_service(service_name="Service 6")
with freeze_time("2019-11-30 15:00:00.000000"): with freeze_time("2019-11-30 15:00:00.000000"):
template_6 = create_template(service_6) template_6 = create_template(service_6)
for x in range(0, 2): for x in range(0, 4):
create_notification(template_6, status="permanent-failure") create_notification(template_6, status="permanent-failure") # notifications too old are excluded
service_2 = create_service(service_name="Service 2") # below threshold service_2 = create_service(service_name="Service 2")
template_2 = create_template(service_2) template_2 = create_template(service_2)
create_notification(template_2, status="permanent-failure", key_type='test') # test key type for x in range(0, 4):
create_notification(template_2, status="permanent-failure", key_type='test') # test key type is excluded
create_notification(template_2, status="permanent-failure") # below threshold is excluded
start_date = (datetime.utcnow() - timedelta(days=1)) start_date = (datetime.utcnow() - timedelta(days=1))
end_date = datetime.utcnow() end_date = datetime.utcnow()
result = dao_find_real_sms_notification_count_by_status_for_live_services(start_date, end_date) result = dao_find_services_with_high_failure_rates(start_date, end_date, threshold=3)
assert len(result) == 3 # assert len(result) == 3
# assert str(result[0].service_id) == fake_uuid
assert len(result) == 1
assert str(result[0].service_id) == fake_uuid assert str(result[0].service_id) == fake_uuid
assert result[0].permanent_failure_rate == 0.25

23
tests/app/service/test_statistics.py
View File

@@ -11,8 +11,7 @@ from app.service.statistics import (
create_stats_dict, create_stats_dict,
create_zeroed_stats_dicts, create_zeroed_stats_dicts,
format_admin_stats, format_admin_stats,
format_statistics, format_statistics
get_rate_of_permanent_failures_for_service
) )
StatsRow = collections.namedtuple('row', ('notification_type', 'status', 'count')) StatsRow = collections.namedtuple('row', ('notification_type', 'status', 'count'))
@@ -74,26 +73,6 @@ def test_format_statistics(stats, email_counts, sms_counts, letter_counts):
} }
@pytest.mark.idparametrize("statistics, expected_result", {
'counts_rate_for_sms': ([
StatsRow('sms', 'permanent-failure', 100),
StatsRow('sms', 'delivered', 300),
], 0.25),
'only_counts_permanent_failure_as_failed': ([
StatsRow('sms', 'permanent-failure', 100),
StatsRow('sms', 'temporary-failure', 100),
StatsRow('sms', 'delivered', 300),
], 0.2),
'below_threshold': ([
StatsRow('sms', 'permanent-failure', 5),
StatsRow('sms', 'delivered', 3),
], 0),
})
def test_get_rate_of_permanent_failures_for_service(statistics, expected_result):
rate = get_rate_of_permanent_failures_for_service(statistics)
assert rate == expected_result
def test_create_zeroed_stats_dicts(): def test_create_zeroed_stats_dicts():
assert create_zeroed_stats_dicts() == { assert create_zeroed_stats_dicts() == {
'sms': {'requested': 0, 'delivered': 0, 'failed': 0}, 'sms': {'requested': 0, 'delivered': 0, 'failed': 0},

37
tests/app/service/test_utils.py
View File

@@ -1,8 +1,5 @@
from app.dao.date_util import get_current_financial_year_start_year from app.dao.date_util import get_current_financial_year_start_year
from freezegun import freeze_time from freezegun import freeze_time
from app.service.utils import get_services_with_high_failure_rates
from collections import namedtuple
from datetime import datetime, timedelta
# see get_financial_year for conversion of financial years. # see get_financial_year for conversion of financial years.
@@ -16,37 +13,3 @@ def test_get_current_financial_year_start_year_before_march():
def test_get_current_financial_year_start_year_after_april(): def test_get_current_financial_year_start_year_after_april():
current_fy = get_current_financial_year_start_year() current_fy = get_current_financial_year_start_year()
assert current_fy == 2017 assert current_fy == 2017
MockServicesNotificationCounts = namedtuple(
'ServicesSendingToTVNumbers',
[
'service_id',
'status',
'count',
]
)
@freeze_time("2019-12-02 12:00:00.000000")
def test_get_services_with_high_failure_rates(mocker, notify_db_session):
mock_query_results = [
MockServicesNotificationCounts('123', 'delivered', 150),
MockServicesNotificationCounts('123', 'permanent-failure', 50), # these will show up
MockServicesNotificationCounts('456', 'delivered', 150),
MockServicesNotificationCounts('456', 'permanent-failure', 5), # ratio too low
MockServicesNotificationCounts('789', 'permanent-failure', 5), # below threshold
MockServicesNotificationCounts('444', 'delivered', 100),
MockServicesNotificationCounts('444', 'permanent-failure', 100), # these will show up
]
mocker.patch(
'app.service.utils.dao_find_real_sms_notification_count_by_status_for_live_services',
return_value=mock_query_results
)
start_date = (datetime.utcnow() - timedelta(days=1))
end_date = datetime.utcnow()
assert get_services_with_high_failure_rates(start_date, end_date) == [
{'id': '123', 'permanent_failure_rate': 0.25},
{'id': '444', 'permanent_failure_rate': 0.5}
]