From b51932179854844f291ad369e23c331087794e3c Mon Sep 17 00:00:00 2001 From: Martyn Inglis Date: Thu, 11 May 2017 15:22:43 +0100 Subject: [PATCH] Adds a query to timeout the job counts. After three days we timeout the notifications that we have not received a receipt for. In the same way we bump the failed count to match the job count if there is a descepenecy. We do this after the same period that we do for notifications. --- app/dao/statistics_dao.py | 25 ++++++- tests/app/dao/test_statistics_dao.py | 101 ++++++++++++++++++++++++++- 2 files changed, 123 insertions(+), 3 deletions(-) diff --git a/app/dao/statistics_dao.py b/app/dao/statistics_dao.py index 9d9044b9b..ca5e134a9 100644 --- a/app/dao/statistics_dao.py +++ b/app/dao/statistics_dao.py @@ -1,4 +1,4 @@ -from datetime import datetime +from datetime import datetime, timedelta from flask import current_app from sqlalchemy.exc import IntegrityError, SQLAlchemyError @@ -16,6 +16,29 @@ from app.models import ( from app.statsd_decorators import statsd +@transactional +def timeout_job_counts(notifications_type, timeout_start): + sent = columns(notifications_type, 'sent') + delivered = columns(notifications_type, 'delivered') + failed = columns(notifications_type, 'failed') + + query = JobStatistics.query.filter( + JobStatistics.created_at < timeout_start, + sent != failed + delivered + ) + return query.update( + {failed: sent - delivered}, synchronize_session=False + ) + + +@statsd(namespace="dao") +def timeout_job_statistics(timeout_period): + timeout_start = datetime.utcnow() - timedelta(seconds=timeout_period) + sms_count = timeout_job_counts(SMS_TYPE, timeout_start) + email_count = timeout_job_counts(EMAIL_TYPE, timeout_start) + return sms_count + email_count + + @statsd(namespace="dao") def create_or_update_job_sending_statistics(notification): if __update_job_stats_sent_count(notification) == 0: diff --git a/tests/app/dao/test_statistics_dao.py b/tests/app/dao/test_statistics_dao.py index a1284575a..7aa0b0fd8 100644 --- a/tests/app/dao/test_statistics_dao.py +++ b/tests/app/dao/test_statistics_dao.py @@ -1,3 +1,4 @@ +from datetime import datetime, timedelta from unittest.mock import call import pytest @@ -5,8 +6,8 @@ from sqlalchemy.exc import IntegrityError, SQLAlchemyError from app.dao.statistics_dao import ( create_or_update_job_sending_statistics, - update_job_stats_outcome_count -) + update_job_stats_outcome_count, + timeout_job_statistics) from app.models import ( JobStatistics, SMS_TYPE, @@ -613,3 +614,99 @@ def test_updating_one_type_of_notification_to_error_maintains_other_counts( assert stats[0].sms_delivered == 0 assert stats[0].sms_failed == 1 assert stats[0].emails_failed == 1 + + +def test_will_timeout_job_counts_after_notification_timeouts(notify_db, notify_db_session, sample_job): + sms_template = sample_template(notify_db, notify_db_session, service=sample_job.service) + email_template = sample_email_template(notify_db, notify_db_session, service=sample_job.service) + + one_minute_ago = datetime.utcnow() - timedelta(minutes=1) + + sms = sample_notification( + notify_db, + notify_db_session, + service=sample_job.service, + template=sms_template, + job=sample_job, + status=NOTIFICATION_CREATED + ) + + email = sample_notification( + notify_db, + notify_db_session, + service=sample_job.service, + template=email_template, + job=sample_job, + status=NOTIFICATION_CREATED + ) + + create_or_update_job_sending_statistics(email) + create_or_update_job_sending_statistics(sms) + + JobStatistics.query.update({JobStatistics.created_at: one_minute_ago}) + + intial_stats = JobStatistics.query.all() + + assert intial_stats[0].emails_sent == 1 + assert intial_stats[0].sms_sent == 1 + assert intial_stats[0].emails_delivered == 0 + assert intial_stats[0].sms_delivered == 0 + assert intial_stats[0].sms_failed == 0 + assert intial_stats[0].emails_failed == 0 + + timeout_job_statistics(59) + updated_stats = JobStatistics.query.all() + assert updated_stats[0].emails_sent == 1 + assert updated_stats[0].sms_sent == 1 + assert updated_stats[0].emails_delivered == 0 + assert updated_stats[0].sms_delivered == 0 + assert updated_stats[0].sms_failed == 1 + assert updated_stats[0].emails_failed == 1 + + +def test_will_not_timeout_job_counts_before_notification_timeouts(notify_db, notify_db_session, sample_job): + sms_template = sample_template(notify_db, notify_db_session, service=sample_job.service) + email_template = sample_email_template(notify_db, notify_db_session, service=sample_job.service) + + one_minute_ago = datetime.utcnow() - timedelta(minutes=1) + + sms = sample_notification( + notify_db, + notify_db_session, + service=sample_job.service, + template=sms_template, + job=sample_job, + status=NOTIFICATION_CREATED + ) + + email = sample_notification( + notify_db, + notify_db_session, + service=sample_job.service, + template=email_template, + job=sample_job, + status=NOTIFICATION_CREATED + ) + + create_or_update_job_sending_statistics(email) + create_or_update_job_sending_statistics(sms) + + JobStatistics.query.update({JobStatistics.created_at: one_minute_ago}) + + intial_stats = JobStatistics.query.all() + + assert intial_stats[0].emails_sent == 1 + assert intial_stats[0].sms_sent == 1 + assert intial_stats[0].emails_delivered == 0 + assert intial_stats[0].sms_delivered == 0 + assert intial_stats[0].sms_failed == 0 + assert intial_stats[0].emails_failed == 0 + + timeout_job_statistics(61) + updated_stats = JobStatistics.query.all() + assert updated_stats[0].emails_sent == 1 + assert updated_stats[0].sms_sent == 1 + assert updated_stats[0].emails_delivered == 0 + assert updated_stats[0].sms_delivered == 0 + assert updated_stats[0].sms_failed == 0 + assert updated_stats[0].emails_failed == 0