diff --git a/app/celery/tasks.py b/app/celery/tasks.py index 5b92db0a5..9935ad609 100644 --- a/app/celery/tasks.py +++ b/app/celery/tasks.py @@ -14,7 +14,8 @@ from notifications_utils.template import ( ) from requests import ( HTTPError, - request + request, + RequestException ) from sqlalchemy.exc import SQLAlchemyError from app import ( @@ -492,25 +493,24 @@ def send_inbound_sms_to_service(self, inbound_sms_id, service_id): "date_received": inbound_sms.provider_date.strftime(DATETIME_FORMAT) } - response = request( - method="POST", - url=inbound_api.url, - data=json.dumps(data), - headers={ - 'Content-Type': 'application/json', - 'Authorization': 'Bearer {}'.format(inbound_api.bearer_token) - }, - timeout=60 - ) - current_app.logger.info('send_inbound_sms_to_service sending {} to {}, response {}'.format( - inbound_sms_id, - inbound_api.url, - response.status_code - )) - try: + response = request( + method="POST", + url=inbound_api.url, + data=json.dumps(data), + headers={ + 'Content-Type': 'application/json', + 'Authorization': 'Bearer {}'.format(inbound_api.bearer_token) + }, + timeout=60 + ) + current_app.logger.info('send_inbound_sms_to_service sending {} to {}, response {}'.format( + inbound_sms_id, + inbound_api.url, + response.status_code + )) response.raise_for_status() - except HTTPError as e: + except RequestException as e: current_app.logger.warning( "send_inbound_sms_to_service request failed for service_id: {} and url: {}. exc: {}".format( service_id, @@ -518,7 +518,7 @@ def send_inbound_sms_to_service(self, inbound_sms_id, service_id): e ) ) - if e.response.status_code >= 500: + if not isinstance(e, HTTPError) or e.response.status_code >= 500: try: self.retry(queue=QueueNames.RETRY, exc='Unable to send_inbound_sms_to_service for service_id: {} and url: {}. \n{}'.format( diff --git a/app/config.py b/app/config.py index c5a656999..8d7ea4ac7 100644 --- a/app/config.py +++ b/app/config.py @@ -245,7 +245,7 @@ class Config(object): 'options': {'queue': QueueNames.PERIODIC} }, 'daily-stats-template-usage-by-month': { - 'task': 'daily-stats-template_usage_by_month', + 'task': 'daily-stats-template-usage-by-month', 'schedule': crontab(hour=0, minute=50), 'options': {'queue': QueueNames.PERIODIC} } diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 0fe8be7d0..0b91e2267 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -554,6 +554,7 @@ def dao_fetch_monthly_historical_usage_by_template_for_service(service_id, year) for result in results: stat = type("", (), {})() stat.template_id = result.template_id + stat.template_type = result.template_type stat.name = str(result.name) stat.month = result.month stat.year = result.year @@ -567,16 +568,18 @@ def dao_fetch_monthly_historical_usage_by_template_for_service(service_id, year) today_results = db.session.query( Notification.template_id, Template.name, + Template.template_type, extract('month', month).label('month'), extract('year', year).label('year'), func.count().label('count') ).join( - Template, Notification.template_id == Template.id + Template, Notification.template_id == Template.id, ).filter( Notification.created_at >= start_date ).group_by( Notification.template_id, Template.name, + Template.template_type, month, year ).order_by( @@ -586,13 +589,15 @@ def dao_fetch_monthly_historical_usage_by_template_for_service(service_id, year) for today_result in today_results: add_to_stats = True for stat in stats: - if today_result.template_id == stat.template_id: + if today_result.template_id == stat.template_id and today_result.month == stat.month \ + and today_result.year == stat.year: stat.count = stat.count + today_result.count add_to_stats = False if add_to_stats: new_stat = type("", (), {})() new_stat.template_id = today_result.template_id + new_stat.template_type = today_result.template_type new_stat.name = today_result.name new_stat.month = today_result.month new_stat.year = today_result.year diff --git a/app/dao/stats_template_usage_by_month_dao.py b/app/dao/stats_template_usage_by_month_dao.py index bacc7a900..84d2550a6 100644 --- a/app/dao/stats_template_usage_by_month_dao.py +++ b/app/dao/stats_template_usage_by_month_dao.py @@ -34,6 +34,7 @@ def dao_get_template_usage_stats_by_service(service_id, year): return db.session.query( StatsTemplateUsageByMonth.template_id, Template.name, + Template.template_type, StatsTemplateUsageByMonth.month, StatsTemplateUsageByMonth.year, StatsTemplateUsageByMonth.count diff --git a/app/service/rest.py b/app/service/rest.py index 250430d7f..b4f25287f 100644 --- a/app/service/rest.py +++ b/app/service/rest.py @@ -539,6 +539,7 @@ def get_monthly_template_usage(service_id): { 'template_id': str(i.template_id), 'name': i.name, + 'type': i.template_type, 'month': i.month, 'year': i.year, 'count': i.count diff --git a/migrations/versions/0139_migrate_sms_allowance_data.py b/migrations/versions/0139_migrate_sms_allowance_data.py new file mode 100644 index 000000000..8fc0d0afa --- /dev/null +++ b/migrations/versions/0139_migrate_sms_allowance_data.py @@ -0,0 +1,50 @@ +""" + +Revision ID: 0139_migrate_sms_allowance_data.py +Revises: 0138_sms_sender_nullable.py +Create Date: 2017-11-10 21:42:59.715203 + +""" +from datetime import datetime +from alembic import op +import uuid +from app.dao.date_util import get_current_financial_year_start_year + + +revision = '0139_migrate_sms_allowance_data' +down_revision = '0138_sms_sender_nullable' + + +def upgrade(): + current_year = get_current_financial_year_start_year() + default_limit = 250000 + + # Step 1: update the column free_sms_fragment_limit in service table if it is empty + update_service_table = """ + UPDATE services SET free_sms_fragment_limit = {} where free_sms_fragment_limit is null + """.format(default_limit) + + op.execute(update_service_table) + + # Step 2: insert at least one row for every service in current year if none exist for that service + insert_row_if_not_exist = """ + INSERT INTO annual_billing + (id, service_id, financial_year_start, free_sms_fragment_limit, created_at, updated_at) + SELECT uuid_in(md5(random()::text)::cstring), id, {}, {}, '{}', '{}' + FROM services WHERE id NOT IN + (select service_id from annual_billing) + """.format(current_year, default_limit, datetime.utcnow(), datetime.utcnow()) + op.execute(insert_row_if_not_exist) + + # Step 3: copy the free_sms_fragment_limit data from the services table across to annual_billing table. + update_sms_allowance = """ + UPDATE annual_billing SET free_sms_fragment_limit = services.free_sms_fragment_limit + FROM services + WHERE annual_billing.service_id = services.id + """ + op.execute(update_sms_allowance) + + +def downgrade(): + # There is no schema change. Only data migration and filling in gaps. + print('There is no action for downgrading to the previous version.') \ No newline at end of file diff --git a/tests/app/celery/test_tasks.py b/tests/app/celery/test_tasks.py index 69f32c421..5267dd436 100644 --- a/tests/app/celery/test_tasks.py +++ b/tests/app/celery/test_tasks.py @@ -7,6 +7,7 @@ import pytest import requests_mock from flask import current_app from freezegun import freeze_time +from requests import RequestException from sqlalchemy.exc import SQLAlchemyError from notifications_utils.template import SMSMessageTemplate, WithSubjectTemplate, LetterDVLATemplate from celery.exceptions import Retry @@ -1160,6 +1161,26 @@ def test_send_inbound_sms_to_service_retries_if_request_returns_500(notify_api, assert exc_msg in mocked.call_args[1]['exc'] +def test_send_inbound_sms_to_service_retries_if_request_throws_unknown(notify_api, sample_service, mocker): + inbound_api = create_service_inbound_api(service=sample_service, url="https://some.service.gov.uk/", + bearer_token="something_unique") + inbound_sms = create_inbound_sms(service=sample_service, notify_number="0751421", user_number="447700900111", + provider_date=datetime(2017, 6, 20), content="Here is some content") + + mocked = mocker.patch('app.celery.tasks.send_inbound_sms_to_service.retry') + mocker.patch("app.celery.tasks.request", side_effect=RequestException()) + + send_inbound_sms_to_service(inbound_sms.id, inbound_sms.service_id) + + exc_msg = 'Unable to send_inbound_sms_to_service for service_id: {} and url: {url}'.format( + sample_service.id, + url=inbound_api.url + ) + assert mocked.call_count == 1 + assert mocked.call_args[1]['queue'] == 'retry-tasks' + assert exc_msg in mocked.call_args[1]['exc'] + + def test_send_inbound_sms_to_service_does_not_retries_if_request_returns_404(notify_api, sample_service, mocker): inbound_api = create_service_inbound_api(service=sample_service, url="https://some.service.gov.uk/", bearer_token="something_unique") diff --git a/tests/app/dao/test_services_dao.py b/tests/app/dao/test_services_dao.py index f880ba4e3..05c564c02 100644 --- a/tests/app/dao/test_services_dao.py +++ b/tests/app/dao/test_services_dao.py @@ -1069,11 +1069,15 @@ def test_dao_fetch_monthly_historical_usage_by_template_for_service_no_stats_tod assert len(result) == 2 assert result[0].template_id == template_two.id + assert result[0].name == template_two.name + assert result[0].template_type == template_two.template_type assert result[0].month == 4 assert result[0].year == 2017 assert result[0].count == 2 assert result[1].template_id == template_one.id + assert result[1].name == template_one.name + assert result[1].template_type == template_two.template_type assert result[1].month == 10 assert result[1].year == 2017 assert result[1].count == 1 @@ -1115,11 +1119,15 @@ def test_dao_fetch_monthly_historical_usage_by_template_for_service_add_to_histo assert len(result) == 2 assert result[0].template_id == template_one.id + assert result[0].name == template_one.name + assert result[0].template_type == template_one.template_type assert result[0].month == 9 assert result[0].year == 2017 assert result[0].count == 1 assert result[1].template_id == template_two.id + assert result[1].name == template_two.name + assert result[1].template_type == template_two.template_type assert result[1].month == 11 assert result[1].year == 2017 assert result[1].count == 2 @@ -1147,16 +1155,22 @@ def test_dao_fetch_monthly_historical_usage_by_template_for_service_add_to_histo assert len(result) == 3 assert result[0].template_id == template_one.id + assert result[0].name == template_one.name + assert result[0].template_type == template_one.template_type assert result[0].month == 9 assert result[0].year == 2017 assert result[0].count == 1 assert result[1].template_id == template_two.id + assert result[1].name == template_two.name + assert result[1].template_type == template_two.template_type assert result[1].month == month assert result[1].year == year assert result[1].count == 3 assert result[2].template_id == template_three.id + assert result[2].name == template_three.name + assert result[2].template_type == template_three.template_type assert result[2].month == 11 assert result[2].year == 2017 assert result[2].count == 1 @@ -1175,9 +1189,9 @@ def test_dao_fetch_monthly_historical_usage_by_template_for_service_does_add_old status='delivered' ) - template_one = create_sample_template(notify_db, notify_db_session, template_name='1') - template_two = create_sample_template(notify_db, notify_db_session, template_name='2') - template_three = create_sample_template(notify_db, notify_db_session, template_name='3') + template_one = create_sample_template(notify_db, notify_db_session, template_name='1', template_type='email') + template_two = create_sample_template(notify_db, notify_db_session, template_name='2', template_type='sms') + template_three = create_sample_template(notify_db, notify_db_session, template_name='3', template_type='letter') date = datetime.now() day = date.day @@ -1198,11 +1212,15 @@ def test_dao_fetch_monthly_historical_usage_by_template_for_service_does_add_old assert len(result) == 2 assert result[0].template_id == template_one.id + assert result[0].name == template_one.name + assert result[0].template_type == template_one.template_type assert result[0].month == 9 assert result[0].year == 2017 assert result[0].count == 1 assert result[1].template_id == template_two.id + assert result[1].name == template_two.name + assert result[1].template_type == template_two.template_type assert result[1].month == 11 assert result[1].year == 2017 assert result[1].count == 2 @@ -1236,9 +1254,9 @@ def test_dao_fetch_monthly_historical_usage_by_template_for_service_get_this_yea status='delivered' ) - template_one = create_sample_template(notify_db, notify_db_session, template_name='1') - template_two = create_sample_template(notify_db, notify_db_session, template_name='2') - template_three = create_sample_template(notify_db, notify_db_session, template_name='3') + template_one = create_sample_template(notify_db, notify_db_session, template_name='1', template_type='email') + template_two = create_sample_template(notify_db, notify_db_session, template_name='2', template_type='sms') + template_three = create_sample_template(notify_db, notify_db_session, template_name='3', template_type='letter') date = datetime.now() day = date.day @@ -1259,6 +1277,8 @@ def test_dao_fetch_monthly_historical_usage_by_template_for_service_get_this_yea assert len(result) == 1 assert result[0].template_id == template_two.id + assert result[0].name == template_two.name + assert result[0].template_type == template_two.template_type assert result[0].month == 11 assert result[0].year == 2017 assert result[0].count == 2 @@ -1292,3 +1312,71 @@ def test_dao_fetch_monthly_historical_usage_by_template_for_service_get_this_yea ) assert len(result) == 2 + + +@freeze_time("2017-11-10 11:09:00.000000") +def test_dao_fetch_monthly_historical_usage_by_template_for_service_combined_historical_current( + notify_db, + notify_db_session, + sample_service +): + notification_history = functools.partial( + create_notification_history, + notify_db, + notify_db_session, + status='delivered' + ) + + template_one = create_sample_template(notify_db, notify_db_session, template_name='1') + + date = datetime.now() + day = date.day + month = date.month + year = date.year + + n = notification_history(created_at=datetime(year, month, day) - timedelta(days=30), sample_template=template_one) + + daily_stats_template_usage_by_month() + + result = sorted( + dao_fetch_monthly_historical_usage_by_template_for_service(n.service_id, 2017), + key=lambda x: (x.month, x.year) + ) + + assert len(result) == 1 + + assert result[0].template_id == template_one.id + assert result[0].name == template_one.name + assert result[0].template_type == template_one.template_type + assert result[0].month == 10 + assert result[0].year == 2017 + assert result[0].count == 1 + + create_notification( + notify_db, + notify_db_session, + service=sample_service, + template=template_one, + created_at=datetime.utcnow() + ) + + result = sorted( + dao_fetch_monthly_historical_usage_by_template_for_service(n.service_id, 2017), + key=lambda x: (x.month, x.year) + ) + + assert len(result) == 2 + + assert result[0].template_id == template_one.id + assert result[0].name == template_one.name + assert result[0].template_type == template_one.template_type + assert result[0].month == 10 + assert result[0].year == 2017 + assert result[0].count == 1 + + assert result[1].template_id == template_one.id + assert result[1].name == template_one.name + assert result[1].template_type == template_one.template_type + assert result[1].month == 11 + assert result[1].year == 2017 + assert result[1].count == 1 diff --git a/tests/app/dao/test_stats_template_usage_by_month_dao.py b/tests/app/dao/test_stats_template_usage_by_month_dao.py index cd21e3af6..99112c6ca 100644 --- a/tests/app/dao/test_stats_template_usage_by_month_dao.py +++ b/tests/app/dao/test_stats_template_usage_by_month_dao.py @@ -99,6 +99,7 @@ def test_dao_get_template_usage_stats_by_service_specific_year(sample_service): assert len(result) == 1 assert result[0].template_id == email_template.id assert result[0].name == email_template.name + assert result[0].template_type == email_template.template_type assert result[0].month == 2 assert result[0].year == 2017 assert result[0].count == 10 diff --git a/tests/app/service/test_rest.py b/tests/app/service/test_rest.py index f7655be72..fad550593 100644 --- a/tests/app/service/test_rest.py +++ b/tests/app/service/test_rest.py @@ -1865,6 +1865,7 @@ def test_get_template_stats_by_month_returns_correct_data(notify_db, notify_db_s assert resp_json["2016-05"][str(sample_template.id)]["counts"]["permanent-failure"] == 1 +@freeze_time('2017-11-11 02:00') def test_get_template_usage_by_month_returns_correct_data( notify_db, notify_db_session, @@ -1918,15 +1919,24 @@ def test_get_template_usage_by_month_returns_correct_data( resp_json = json.loads(resp.get_data(as_text=True)).get('stats') assert resp.status_code == 200 - assert len(resp_json) == 1 + assert len(resp_json) == 2 assert resp_json[0]["template_id"] == str(sample_template.id) assert resp_json[0]["name"] == sample_template.name + assert resp_json[0]["type"] == sample_template.template_type assert resp_json[0]["month"] == 4 assert resp_json[0]["year"] == 2016 - assert resp_json[0]["count"] == 5 + assert resp_json[0]["count"] == 4 + + assert resp_json[1]["template_id"] == str(sample_template.id) + assert resp_json[1]["name"] == sample_template.name + assert resp_json[1]["type"] == sample_template.template_type + assert resp_json[1]["month"] == 11 + assert resp_json[1]["year"] == 2017 + assert resp_json[1]["count"] == 1 +@freeze_time('2017-11-11 02:00') def test_get_template_usage_by_month_returns_two_templates( notify_db, notify_db_session, @@ -1983,19 +1993,30 @@ def test_get_template_usage_by_month_returns_two_templates( resp_json = json.loads(resp.get_data(as_text=True)).get('stats') assert resp.status_code == 200 - assert len(resp_json) == 2 + assert len(resp_json) == 3 - resp_json = sorted(resp_json, key=lambda k: k.get('count', 0)) + resp_json = sorted(resp_json, key=lambda k: (k.get('year', 0), k.get('month', 0), k.get('count', 0))) assert resp_json[0]["template_id"] == str(template_one.id) + assert resp_json[0]["name"] == template_one.name + assert resp_json[0]["type"] == template_one.template_type assert resp_json[0]["month"] == 4 assert resp_json[0]["year"] == 2016 assert resp_json[0]["count"] == 1 assert resp_json[1]["template_id"] == str(sample_template.id) + assert resp_json[1]["name"] == sample_template.name + assert resp_json[1]["type"] == sample_template.template_type assert resp_json[1]["month"] == 4 assert resp_json[1]["year"] == 2016 - assert resp_json[1]["count"] == 4 + assert resp_json[1]["count"] == 3 + + assert resp_json[2]["template_id"] == str(sample_template.id) + assert resp_json[2]["name"] == sample_template.name + assert resp_json[2]["type"] == sample_template.template_type + assert resp_json[2]["month"] == 11 + assert resp_json[2]["year"] == 2017 + assert resp_json[2]["count"] == 1 @pytest.mark.parametrize('query_string, expected_status, expected_json', [