diff --git a/app/dao/notification_usage_dao.py b/app/dao/notification_usage_dao.py deleted file mode 100644 index 7105f5b3e..000000000 --- a/app/dao/notification_usage_dao.py +++ /dev/null @@ -1,180 +0,0 @@ -from datetime import datetime, timedelta - -from notifications_utils.statsd_decorators import statsd -from sqlalchemy import Float, Integer, and_ -from sqlalchemy import func, case, cast -from sqlalchemy import literal_column - -from app import db -from app.dao.date_util import get_financial_year -from app.models import ( - NotificationHistory, - Rate, - NOTIFICATION_STATUS_TYPES_BILLABLE, - KEY_TYPE_TEST, - SMS_TYPE, - EMAIL_TYPE, - LETTER_TYPE, - LetterRate, - Service -) -from app.utils import get_london_month_from_utc_column - - -@statsd(namespace="dao") -def get_billing_data_for_month(service_id, start_date, end_date, notification_type): - results = [] - - if notification_type == EMAIL_TYPE: - return billing_data_per_month_query(0, service_id, start_date, end_date, EMAIL_TYPE) - - elif notification_type == SMS_TYPE: - rates = get_rates_for_daterange(start_date, end_date, SMS_TYPE) - - if not rates: - return [] - - # so the start end date in the query are the valid from the rate, not the month - # - this is going to take some thought - for r, n in zip(rates, rates[1:]): - results.extend( - billing_data_per_month_query( - r.rate, service_id, max(r.valid_from, start_date), - min(n.valid_from, end_date), SMS_TYPE - ) - ) - results.extend( - billing_data_per_month_query( - rates[-1].rate, service_id, max(rates[-1].valid_from, start_date), - end_date, SMS_TYPE - ) - ) - elif notification_type == LETTER_TYPE: - results.extend(billing_letter_data_per_month_query(service_id, start_date, end_date)) - - return results - - -@statsd(namespace="dao") -def get_monthly_billing_data(service_id, year): - start_date, end_date = get_financial_year(year) - rates = get_rates_for_daterange(start_date, end_date, SMS_TYPE) - - if not rates: - return [] - - result = [] - for r, n in zip(rates, rates[1:]): - result.extend(billing_data_per_month_query(r.rate, service_id, r.valid_from, n.valid_from, SMS_TYPE)) - result.extend(billing_data_per_month_query(rates[-1].rate, service_id, rates[-1].valid_from, end_date, SMS_TYPE)) - - return [(datetime.strftime(x[0], "%B"), x[1], x[2], x[3], x[4], x[5]) for x in result] - - -def billing_data_filter(notification_type, start_date, end_date, service_id): - return [ - NotificationHistory.notification_type == notification_type, - NotificationHistory.created_at.between(start_date, end_date), - NotificationHistory.service_id == service_id, - NotificationHistory.status.in_(NOTIFICATION_STATUS_TYPES_BILLABLE), - NotificationHistory.key_type != KEY_TYPE_TEST - ] - - -def get_rates_for_daterange(start_date, end_date, notification_type): - rates = Rate.query.filter(Rate.notification_type == notification_type).order_by(Rate.valid_from).all() - - if not rates: - return [] - - results = [] - for current_rate, current_rate_expiry_date in zip(rates, rates[1:]): - if is_between(current_rate.valid_from, start_date, end_date) or \ - is_between(current_rate_expiry_date.valid_from - timedelta(microseconds=1), start_date, end_date): - results.append(current_rate) - - if is_between(rates[-1].valid_from, start_date, end_date): - results.append(rates[-1]) - - if not results: - for x in reversed(rates): - if start_date >= x.valid_from: - results.append(x) - break - - return results - - -def is_between(date, start_date, end_date): - return start_date <= date <= end_date - - -@statsd(namespace="dao") -def billing_data_per_month_query(rate, service_id, start_date, end_date, notification_type): - month = get_london_month_from_utc_column(NotificationHistory.created_at) - if notification_type == SMS_TYPE: - filter_subq = func.sum(NotificationHistory.billable_units).label('billing_units') - elif notification_type == EMAIL_TYPE: - filter_subq = func.count(NotificationHistory.billable_units).label('billing_units') - - results = db.session.query( - month.label('month'), - filter_subq, - rate_multiplier().label('rate_multiplier'), - NotificationHistory.international, - NotificationHistory.notification_type, - cast(rate, Float()).label('rate') - ).filter( - *billing_data_filter(notification_type, start_date, end_date, service_id) - ).group_by( - NotificationHistory.notification_type, - month, - NotificationHistory.rate_multiplier, - NotificationHistory.international - ).order_by( - month, - rate_multiplier() - ) - return results.all() - - -def rate_multiplier(): - return cast(case([ - (NotificationHistory.rate_multiplier == None, literal_column("'1'")), # noqa - (NotificationHistory.rate_multiplier != None, NotificationHistory.rate_multiplier), # noqa - ]), Integer()) - - -@statsd(namespace="dao") -def billing_letter_data_per_month_query(service_id, start_date, end_date): - month = get_london_month_from_utc_column(NotificationHistory.created_at) - crown = Service.query.get(service_id).crown - results = db.session.query( - month.label('month'), - func.count(NotificationHistory.billable_units).label('billing_units'), - rate_multiplier().label('rate_multiplier'), - NotificationHistory.international, - NotificationHistory.notification_type, - cast(LetterRate.rate, Float()).label('rate') - ).join( - LetterRate, - and_(NotificationHistory.created_at >= LetterRate.start_date, - (LetterRate.end_date == None) | # noqa - (LetterRate.end_date > NotificationHistory.created_at)) - ).filter( - LetterRate.sheet_count == NotificationHistory.billable_units, - LetterRate.crown == crown, - LetterRate.post_class == 'second', - NotificationHistory.created_at < end_date, - *billing_data_filter(LETTER_TYPE, start_date, end_date, service_id) - ).group_by( - NotificationHistory.notification_type, - month, - NotificationHistory.rate_multiplier, - NotificationHistory.international, - LetterRate.rate - ).order_by( - month, - rate_multiplier() - ) - return results.all() diff --git a/app/service/statistics.py b/app/service/statistics.py index d942ad427..5c6accaa5 100644 --- a/app/service/statistics.py +++ b/app/service/statistics.py @@ -84,12 +84,13 @@ def create_zeroed_stats_dicts(): def _update_statuses_from_row(update_dict, row): - update_dict['requested'] += row.count + if row.status != 'cancelled': + update_dict['requested'] += row.count if row.status in ('delivered', 'sent'): update_dict['delivered'] += row.count elif row.status in ( 'failed', 'technical-failure', 'temporary-failure', - 'permanent-failure', 'validation-failed', 'virus-scan-failed', 'cancelled'): + 'permanent-failure', 'validation-failed', 'virus-scan-failed'): update_dict['failed'] += row.count diff --git a/migrations/versions/0249_another_letter_org.py b/migrations/versions/0249_another_letter_org.py new file mode 100644 index 000000000..e4423ede8 --- /dev/null +++ b/migrations/versions/0249_another_letter_org.py @@ -0,0 +1,35 @@ +"""empty message + +Revision ID: 0249_another_letter_org +Revises: 0248_enable_choose_postage + +""" + +# revision identifiers, used by Alembic. +revision = '0249_another_letter_org' +down_revision = '0248_enable_choose_postage' + +from alembic import op + + +NEW_ORGANISATIONS = [ + ('521', 'North Somerset Council', 'north-somerset'), +] + + +def upgrade(): + for numeric_id, name, filename in NEW_ORGANISATIONS: + op.execute(""" + INSERT + INTO dvla_organisation + VALUES ('{}', '{}', '{}') + """.format(numeric_id, name, filename)) + + +def downgrade(): + for numeric_id, _, _ in NEW_ORGANISATIONS: + op.execute(""" + DELETE + FROM dvla_organisation + WHERE id = '{}' + """.format(numeric_id)) diff --git a/requirements-app.txt b/requirements-app.txt index f4097456a..c152dff4a 100644 --- a/requirements-app.txt +++ b/requirements-app.txt @@ -29,8 +29,6 @@ awscli-cwlogs>=1.4,<1.5 # Putting upgrade on hold due to v1.0.0 using sha512 instead of sha1 by default itsdangerous==0.24 # pyup: <1.0.0 -git+https://github.com/alphagov/notifications-utils.git@30.7.2#egg=notifications-utils==30.7.2 -# pinned until upgrade to redis-py 3 works -redis==2.10.6 +git+https://github.com/alphagov/notifications-utils.git@30.7.4#egg=notifications-utils==30.7.4 git+https://github.com/alphagov/boto.git@2.43.0-patch3#egg=boto==2.43.0-patch3 diff --git a/requirements.txt b/requirements.txt index e232d5b51..e508e33f8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -31,9 +31,7 @@ awscli-cwlogs>=1.4,<1.5 # Putting upgrade on hold due to v1.0.0 using sha512 instead of sha1 by default itsdangerous==0.24 # pyup: <1.0.0 -git+https://github.com/alphagov/notifications-utils.git@30.7.2#egg=notifications-utils==30.7.2 -# pinned until upgrade to redis-py 3 works -redis==2.10.6 +git+https://github.com/alphagov/notifications-utils.git@30.7.4#egg=notifications-utils==30.7.4 git+https://github.com/alphagov/boto.git@2.43.0-patch3#egg=boto==2.43.0-patch3 @@ -42,12 +40,12 @@ alembic==1.0.5 amqp==1.4.9 anyjson==0.3.3 attrs==18.2.0 -awscli==1.16.84 +awscli==1.16.85 bcrypt==3.1.5 billiard==3.3.0.23 -bleach==2.1.3 +bleach==3.0.2 boto3==1.6.16 -botocore==1.12.74 +botocore==1.12.75 certifi==2018.11.29 chardet==3.0.4 Click==7.0 @@ -56,32 +54,32 @@ docutils==0.14 Flask-Redis==0.3.0 future==0.17.1 greenlet==0.4.15 -html5lib==1.0.1 idna==2.8 Jinja2==2.10 jmespath==0.9.3 kombu==3.0.37 Mako==1.0.7 MarkupSafe==1.1.0 -mistune==0.8.3 +mistune==0.8.4 monotonic==1.5 orderedset==2.0.1 -phonenumbers==8.9.4 +phonenumbers==8.10.2 pyasn1==0.4.5 pycparser==2.19 PyPDF2==1.26.0 pyrsistent==0.14.9 python-dateutil==2.7.5 python-editor==1.0.3 -python-json-logger==0.1.8 +python-json-logger==0.1.10 pytz==2018.9 PyYAML==3.12 +redis==3.0.1 requests==2.21.0 rsa==3.4.2 s3transfer==0.1.13 six==1.12.0 smartypants==2.0.1 -statsd==3.2.2 +statsd==3.3.0 urllib3==1.24.1 webencodings==0.5.1 Werkzeug==0.14.1 diff --git a/tests/app/dao/test_notification_usage_dao.py b/tests/app/dao/test_notification_usage_dao.py deleted file mode 100644 index 4e1bb27ab..000000000 --- a/tests/app/dao/test_notification_usage_dao.py +++ /dev/null @@ -1,235 +0,0 @@ -import uuid -from datetime import datetime, timedelta - -from freezegun import freeze_time - -from app.dao.date_util import get_financial_year -from app.dao.notification_usage_dao import ( - get_rates_for_daterange, - get_billing_data_for_month, - get_monthly_billing_data, - billing_letter_data_per_month_query -) -from app.models import ( - Rate, - SMS_TYPE, -) -from tests.app.db import create_notification, create_rate, create_template, create_service - - -def test_get_rates_for_daterange(notify_db, notify_db_session): - set_up_rate(notify_db, datetime(2016, 5, 18), 0.016) - set_up_rate(notify_db, datetime(2017, 3, 31, 23), 0.0158) - start_date, end_date = get_financial_year(2017) - rates = get_rates_for_daterange(start_date, end_date, SMS_TYPE) - assert len(rates) == 1 - assert datetime.strftime(rates[0].valid_from, '%Y-%m-%d %H:%M:%S') == "2017-03-31 23:00:00" - assert rates[0].rate == 0.0158 - - -def test_get_rates_for_daterange_multiple_result_per_year(notify_db, notify_db_session): - set_up_rate(notify_db, datetime(2016, 4, 1), 0.015) - set_up_rate(notify_db, datetime(2016, 5, 18), 0.016) - set_up_rate(notify_db, datetime(2017, 4, 1), 0.0158) - start_date, end_date = get_financial_year(2016) - rates = get_rates_for_daterange(start_date, end_date, SMS_TYPE) - assert len(rates) == 2 - assert datetime.strftime(rates[0].valid_from, '%Y-%m-%d %H:%M:%S') == "2016-04-01 00:00:00" - assert rates[0].rate == 0.015 - assert datetime.strftime(rates[1].valid_from, '%Y-%m-%d %H:%M:%S') == "2016-05-18 00:00:00" - assert rates[1].rate == 0.016 - - -def test_get_rates_for_daterange_returns_correct_rates(notify_db, notify_db_session): - set_up_rate(notify_db, datetime(2016, 4, 1), 0.015) - set_up_rate(notify_db, datetime(2016, 9, 1), 0.016) - set_up_rate(notify_db, datetime(2017, 6, 1), 0.0175) - start_date, end_date = get_financial_year(2017) - rates_2017 = get_rates_for_daterange(start_date, end_date, SMS_TYPE) - assert len(rates_2017) == 2 - assert datetime.strftime(rates_2017[0].valid_from, '%Y-%m-%d %H:%M:%S') == "2016-09-01 00:00:00" - assert rates_2017[0].rate == 0.016 - assert datetime.strftime(rates_2017[1].valid_from, '%Y-%m-%d %H:%M:%S') == "2017-06-01 00:00:00" - assert rates_2017[1].rate == 0.0175 - - -def test_get_rates_for_daterange_in_the_future(notify_db, notify_db_session): - set_up_rate(notify_db, datetime(2016, 4, 1), 0.015) - set_up_rate(notify_db, datetime(2017, 6, 1), 0.0175) - start_date, end_date = get_financial_year(2018) - rates = get_rates_for_daterange(start_date, end_date, SMS_TYPE) - assert datetime.strftime(rates[0].valid_from, '%Y-%m-%d %H:%M:%S') == "2017-06-01 00:00:00" - assert rates[0].rate == 0.0175 - - -def test_get_rates_for_daterange_returns_empty_list_if_year_is_before_earliest_rate(notify_db, notify_db_session): - set_up_rate(notify_db, datetime(2016, 4, 1), 0.015) - set_up_rate(notify_db, datetime(2017, 6, 1), 0.0175) - start_date, end_date = get_financial_year(2015) - rates = get_rates_for_daterange(start_date, end_date, SMS_TYPE) - assert rates == [] - - -def test_get_rates_for_daterange_early_rate(notify_db, notify_db_session): - set_up_rate(notify_db, datetime(2015, 6, 1), 0.014) - set_up_rate(notify_db, datetime(2016, 6, 1), 0.015) - set_up_rate(notify_db, datetime(2016, 9, 1), 0.016) - set_up_rate(notify_db, datetime(2017, 6, 1), 0.0175) - start_date, end_date = get_financial_year(2016) - rates = get_rates_for_daterange(start_date, end_date, SMS_TYPE) - assert len(rates) == 3 - - -def test_get_rates_for_daterange_edge_case(notify_db, notify_db_session): - set_up_rate(notify_db, datetime(2016, 3, 31, 23, 00), 0.015) - set_up_rate(notify_db, datetime(2017, 3, 31, 23, 00), 0.0175) - start_date, end_date = get_financial_year(2016) - rates = get_rates_for_daterange(start_date, end_date, SMS_TYPE) - assert len(rates) == 1 - assert datetime.strftime(rates[0].valid_from, '%Y-%m-%d %H:%M:%S') == "2016-03-31 23:00:00" - assert rates[0].rate == 0.015 - - -def test_get_rates_for_daterange_where_daterange_is_one_month_that_falls_between_rate_valid_from( - notify_db, notify_db_session -): - set_up_rate(notify_db, datetime(2017, 1, 1), 0.175) - set_up_rate(notify_db, datetime(2017, 3, 31), 0.123) - start_date = datetime(2017, 2, 1, 00, 00, 00) - end_date = datetime(2017, 2, 28, 23, 59, 59, 99999) - rates = get_rates_for_daterange(start_date, end_date, SMS_TYPE) - assert len(rates) == 1 - assert datetime.strftime(rates[0].valid_from, '%Y-%m-%d %H:%M:%S') == "2017-01-01 00:00:00" - assert rates[0].rate == 0.175 - - -def test_get_monthly_billing_data(notify_db, notify_db_session, sample_template, sample_email_template): - set_up_rate(notify_db, datetime(2016, 4, 1), 0.014) - # previous year - create_notification(template=sample_template, created_at=datetime(2016, 3, 31), sent_at=datetime(2016, 3, 31), - status='sending', billable_units=1) - # current year - create_notification(template=sample_template, created_at=datetime(2016, 4, 2), sent_at=datetime(2016, 4, 2), - status='sending', billable_units=1) - create_notification(template=sample_template, created_at=datetime(2016, 5, 18), sent_at=datetime(2016, 5, 18), - status='sending', billable_units=2) - create_notification(template=sample_template, created_at=datetime(2016, 7, 22), sent_at=datetime(2016, 7, 22), - status='sending', billable_units=3) - create_notification(template=sample_template, created_at=datetime(2016, 7, 22), sent_at=datetime(2016, 7, 22), - status='sending', billable_units=3, rate_multiplier=2) - create_notification(template=sample_template, created_at=datetime(2016, 7, 22), sent_at=datetime(2016, 7, 22), - status='sending', billable_units=3, rate_multiplier=2) - create_notification(template=sample_template, created_at=datetime(2016, 7, 30), sent_at=datetime(2016, 7, 22), - status='sending', billable_units=4) - - create_notification(template=sample_email_template, created_at=datetime(2016, 8, 22), sent_at=datetime(2016, 7, 22), - status='sending', billable_units=0) - create_notification(template=sample_email_template, created_at=datetime(2016, 8, 30), sent_at=datetime(2016, 7, 22), - status='sending', billable_units=0) - # next year - create_notification(template=sample_template, created_at=datetime(2017, 3, 31, 23, 00, 00), - sent_at=datetime(2017, 3, 31), status='sending', billable_units=6) - results = get_monthly_billing_data(sample_template.service_id, 2016) - assert len(results) == 4 - # (billable_units, rate_multiplier, international, type, rate) - assert results[0] == ('April', 1, 1, False, SMS_TYPE, 0.014) - assert results[1] == ('May', 2, 1, False, SMS_TYPE, 0.014) - assert results[2] == ('July', 7, 1, False, SMS_TYPE, 0.014) - assert results[3] == ('July', 6, 2, False, SMS_TYPE, 0.014) - - -def test_get_monthly_billing_data_with_multiple_rates(notify_db, notify_db_session, sample_template, - sample_email_template): - set_up_rate(notify_db, datetime(2016, 4, 1), 0.014) - set_up_rate(notify_db, datetime(2016, 6, 5), 0.0175) - set_up_rate(notify_db, datetime(2017, 7, 5), 0.018) - # previous year - create_notification(template=sample_template, created_at=datetime(2016, 3, 31), sent_at=datetime(2016, 3, 31), - status='sending', billable_units=1) - # current year - create_notification(template=sample_template, created_at=datetime(2016, 4, 2), sent_at=datetime(2016, 4, 2), - status='sending', billable_units=1) - create_notification(template=sample_template, created_at=datetime(2016, 5, 18), sent_at=datetime(2016, 5, 18), - status='sending', billable_units=2) - create_notification(template=sample_template, created_at=datetime(2016, 6, 1), sent_at=datetime(2016, 6, 1), - status='sending', billable_units=3) - create_notification(template=sample_template, created_at=datetime(2016, 6, 15), sent_at=datetime(2016, 6, 15), - status='sending', billable_units=4) - create_notification(template=sample_email_template, created_at=datetime(2016, 8, 22), - sent_at=datetime(2016, 7, 22), - status='sending', billable_units=0) - create_notification(template=sample_email_template, created_at=datetime(2016, 8, 30), - sent_at=datetime(2016, 7, 22), - status='sending', billable_units=0) - # next year - create_notification(template=sample_template, created_at=datetime(2017, 3, 31, 23, 00, 00), - sent_at=datetime(2017, 3, 31), status='sending', billable_units=6) - results = get_monthly_billing_data(sample_template.service_id, 2016) - assert len(results) == 4 - assert results[0] == ('April', 1, 1, False, SMS_TYPE, 0.014) - assert results[1] == ('May', 2, 1, False, SMS_TYPE, 0.014) - assert results[2] == ('June', 3, 1, False, SMS_TYPE, 0.014) - assert results[3] == ('June', 4, 1, False, SMS_TYPE, 0.0175) - - -def test_get_monthly_billing_data_with_no_notifications_for_daterange(notify_db, notify_db_session, sample_template): - set_up_rate(notify_db, datetime(2016, 4, 1), 0.014) - results = get_monthly_billing_data(sample_template.service_id, 2016) - assert results == [] - - -def set_up_rate(notify_db, start_date, value): - rate = Rate(id=uuid.uuid4(), valid_from=start_date, rate=value, notification_type=SMS_TYPE) - notify_db.session.add(rate) - - -@freeze_time("2016-05-01") -def test_get_billing_data_for_month_where_start_date_before_rate_returns_empty( - sample_template -): - create_rate(datetime(2016, 4, 1), 0.014, SMS_TYPE) - - results = get_monthly_billing_data( - service_id=sample_template.service_id, - year=2015 - ) - - assert not results - - -@freeze_time("2016-05-01") -def test_get_monthly_billing_data_where_start_date_before_rate_returns_empty( - sample_template -): - now = datetime.utcnow() - create_rate(now, 0.014, SMS_TYPE) - - results = get_billing_data_for_month( - service_id=sample_template.service_id, - start_date=now - timedelta(days=2), - end_date=now - timedelta(days=1), - notification_type=SMS_TYPE - ) - - assert not results - - -def test_billing_letter_data_per_month_query( - notify_db_session -): - service = create_service() - template = create_template(service=service, template_type='letter') - create_notification(template=template, billable_units=1, created_at=datetime(2017, 2, 1, 13, 21), - status='delivered') - create_notification(template=template, billable_units=1, created_at=datetime(2017, 2, 1, 13, 21), - status='delivered') - create_notification(template=template, billable_units=1, created_at=datetime(2017, 2, 1, 13, 21), - status='delivered') - - results = billing_letter_data_per_month_query(service_id=service.id, - start_date=datetime(2017, 2, 1), - end_date=datetime(2017, 2, 28)) - - assert len(results) == 1 - assert results[0].rate == 0.3 - assert results[0].billing_units == 3 diff --git a/tests/app/service/test_statistics.py b/tests/app/service/test_statistics.py index d553b90f2..07a5d5c09 100644 --- a/tests/app/service/test_statistics.py +++ b/tests/app/service/test_statistics.py @@ -39,7 +39,7 @@ NewStatsRow = collections.namedtuple('row', ('notification_type', 'status', 'key StatsRow('letter', 'virus-scan-failed', 1), StatsRow('letter', 'permanent-failure', 1), StatsRow('letter', 'cancelled', 1), - ], [4, 0, 4], [0, 0, 0], [4, 0, 4]), + ], [4, 0, 4], [0, 0, 0], [3, 0, 3]), 'convert_sent_to_delivered': ([ StatsRow('sms', 'sending', 1), StatsRow('sms', 'delivered', 1),