Merge branch 'master' into use-ft-notification-statu-for-monthly-template-usage

This commit is contained in:
Rebecca Law
2019-01-11 17:11:23 +00:00
7 changed files with 49 additions and 432 deletions

View File

@@ -1,180 +0,0 @@
from datetime import datetime, timedelta
from notifications_utils.statsd_decorators import statsd
from sqlalchemy import Float, Integer, and_
from sqlalchemy import func, case, cast
from sqlalchemy import literal_column
from app import db
from app.dao.date_util import get_financial_year
from app.models import (
NotificationHistory,
Rate,
NOTIFICATION_STATUS_TYPES_BILLABLE,
KEY_TYPE_TEST,
SMS_TYPE,
EMAIL_TYPE,
LETTER_TYPE,
LetterRate,
Service
)
from app.utils import get_london_month_from_utc_column
@statsd(namespace="dao")
def get_billing_data_for_month(service_id, start_date, end_date, notification_type):
results = []
if notification_type == EMAIL_TYPE:
return billing_data_per_month_query(0, service_id, start_date, end_date, EMAIL_TYPE)
elif notification_type == SMS_TYPE:
rates = get_rates_for_daterange(start_date, end_date, SMS_TYPE)
if not rates:
return []
# so the start end date in the query are the valid from the rate, not the month
# - this is going to take some thought
for r, n in zip(rates, rates[1:]):
results.extend(
billing_data_per_month_query(
r.rate, service_id, max(r.valid_from, start_date),
min(n.valid_from, end_date), SMS_TYPE
)
)
results.extend(
billing_data_per_month_query(
rates[-1].rate, service_id, max(rates[-1].valid_from, start_date),
end_date, SMS_TYPE
)
)
elif notification_type == LETTER_TYPE:
results.extend(billing_letter_data_per_month_query(service_id, start_date, end_date))
return results
@statsd(namespace="dao")
def get_monthly_billing_data(service_id, year):
start_date, end_date = get_financial_year(year)
rates = get_rates_for_daterange(start_date, end_date, SMS_TYPE)
if not rates:
return []
result = []
for r, n in zip(rates, rates[1:]):
result.extend(billing_data_per_month_query(r.rate, service_id, r.valid_from, n.valid_from, SMS_TYPE))
result.extend(billing_data_per_month_query(rates[-1].rate, service_id, rates[-1].valid_from, end_date, SMS_TYPE))
return [(datetime.strftime(x[0], "%B"), x[1], x[2], x[3], x[4], x[5]) for x in result]
def billing_data_filter(notification_type, start_date, end_date, service_id):
return [
NotificationHistory.notification_type == notification_type,
NotificationHistory.created_at.between(start_date, end_date),
NotificationHistory.service_id == service_id,
NotificationHistory.status.in_(NOTIFICATION_STATUS_TYPES_BILLABLE),
NotificationHistory.key_type != KEY_TYPE_TEST
]
def get_rates_for_daterange(start_date, end_date, notification_type):
rates = Rate.query.filter(Rate.notification_type == notification_type).order_by(Rate.valid_from).all()
if not rates:
return []
results = []
for current_rate, current_rate_expiry_date in zip(rates, rates[1:]):
if is_between(current_rate.valid_from, start_date, end_date) or \
is_between(current_rate_expiry_date.valid_from - timedelta(microseconds=1), start_date, end_date):
results.append(current_rate)
if is_between(rates[-1].valid_from, start_date, end_date):
results.append(rates[-1])
if not results:
for x in reversed(rates):
if start_date >= x.valid_from:
results.append(x)
break
return results
def is_between(date, start_date, end_date):
return start_date <= date <= end_date
@statsd(namespace="dao")
def billing_data_per_month_query(rate, service_id, start_date, end_date, notification_type):
month = get_london_month_from_utc_column(NotificationHistory.created_at)
if notification_type == SMS_TYPE:
filter_subq = func.sum(NotificationHistory.billable_units).label('billing_units')
elif notification_type == EMAIL_TYPE:
filter_subq = func.count(NotificationHistory.billable_units).label('billing_units')
results = db.session.query(
month.label('month'),
filter_subq,
rate_multiplier().label('rate_multiplier'),
NotificationHistory.international,
NotificationHistory.notification_type,
cast(rate, Float()).label('rate')
).filter(
*billing_data_filter(notification_type, start_date, end_date, service_id)
).group_by(
NotificationHistory.notification_type,
month,
NotificationHistory.rate_multiplier,
NotificationHistory.international
).order_by(
month,
rate_multiplier()
)
return results.all()
def rate_multiplier():
return cast(case([
(NotificationHistory.rate_multiplier == None, literal_column("'1'")), # noqa
(NotificationHistory.rate_multiplier != None, NotificationHistory.rate_multiplier), # noqa
]), Integer())
@statsd(namespace="dao")
def billing_letter_data_per_month_query(service_id, start_date, end_date):
month = get_london_month_from_utc_column(NotificationHistory.created_at)
crown = Service.query.get(service_id).crown
results = db.session.query(
month.label('month'),
func.count(NotificationHistory.billable_units).label('billing_units'),
rate_multiplier().label('rate_multiplier'),
NotificationHistory.international,
NotificationHistory.notification_type,
cast(LetterRate.rate, Float()).label('rate')
).join(
LetterRate,
and_(NotificationHistory.created_at >= LetterRate.start_date,
(LetterRate.end_date == None) | # noqa
(LetterRate.end_date > NotificationHistory.created_at))
).filter(
LetterRate.sheet_count == NotificationHistory.billable_units,
LetterRate.crown == crown,
LetterRate.post_class == 'second',
NotificationHistory.created_at < end_date,
*billing_data_filter(LETTER_TYPE, start_date, end_date, service_id)
).group_by(
NotificationHistory.notification_type,
month,
NotificationHistory.rate_multiplier,
NotificationHistory.international,
LetterRate.rate
).order_by(
month,
rate_multiplier()
)
return results.all()

View File

@@ -84,12 +84,13 @@ def create_zeroed_stats_dicts():
def _update_statuses_from_row(update_dict, row):
update_dict['requested'] += row.count
if row.status != 'cancelled':
update_dict['requested'] += row.count
if row.status in ('delivered', 'sent'):
update_dict['delivered'] += row.count
elif row.status in (
'failed', 'technical-failure', 'temporary-failure',
'permanent-failure', 'validation-failed', 'virus-scan-failed', 'cancelled'):
'permanent-failure', 'validation-failed', 'virus-scan-failed'):
update_dict['failed'] += row.count

View File

@@ -0,0 +1,35 @@
"""empty message
Revision ID: 0249_another_letter_org
Revises: 0248_enable_choose_postage
"""
# revision identifiers, used by Alembic.
revision = '0249_another_letter_org'
down_revision = '0248_enable_choose_postage'
from alembic import op
NEW_ORGANISATIONS = [
('521', 'North Somerset Council', 'north-somerset'),
]
def upgrade():
for numeric_id, name, filename in NEW_ORGANISATIONS:
op.execute("""
INSERT
INTO dvla_organisation
VALUES ('{}', '{}', '{}')
""".format(numeric_id, name, filename))
def downgrade():
for numeric_id, _, _ in NEW_ORGANISATIONS:
op.execute("""
DELETE
FROM dvla_organisation
WHERE id = '{}'
""".format(numeric_id))

View File

@@ -29,8 +29,6 @@ awscli-cwlogs>=1.4,<1.5
# Putting upgrade on hold due to v1.0.0 using sha512 instead of sha1 by default
itsdangerous==0.24 # pyup: <1.0.0
git+https://github.com/alphagov/notifications-utils.git@30.7.2#egg=notifications-utils==30.7.2
# pinned until upgrade to redis-py 3 works
redis==2.10.6
git+https://github.com/alphagov/notifications-utils.git@30.7.4#egg=notifications-utils==30.7.4
git+https://github.com/alphagov/boto.git@2.43.0-patch3#egg=boto==2.43.0-patch3

View File

@@ -31,9 +31,7 @@ awscli-cwlogs>=1.4,<1.5
# Putting upgrade on hold due to v1.0.0 using sha512 instead of sha1 by default
itsdangerous==0.24 # pyup: <1.0.0
git+https://github.com/alphagov/notifications-utils.git@30.7.2#egg=notifications-utils==30.7.2
# pinned until upgrade to redis-py 3 works
redis==2.10.6
git+https://github.com/alphagov/notifications-utils.git@30.7.4#egg=notifications-utils==30.7.4
git+https://github.com/alphagov/boto.git@2.43.0-patch3#egg=boto==2.43.0-patch3
@@ -42,12 +40,12 @@ alembic==1.0.5
amqp==1.4.9
anyjson==0.3.3
attrs==18.2.0
awscli==1.16.84
awscli==1.16.85
bcrypt==3.1.5
billiard==3.3.0.23
bleach==2.1.3
bleach==3.0.2
boto3==1.6.16
botocore==1.12.74
botocore==1.12.75
certifi==2018.11.29
chardet==3.0.4
Click==7.0
@@ -56,32 +54,32 @@ docutils==0.14
Flask-Redis==0.3.0
future==0.17.1
greenlet==0.4.15
html5lib==1.0.1
idna==2.8
Jinja2==2.10
jmespath==0.9.3
kombu==3.0.37
Mako==1.0.7
MarkupSafe==1.1.0
mistune==0.8.3
mistune==0.8.4
monotonic==1.5
orderedset==2.0.1
phonenumbers==8.9.4
phonenumbers==8.10.2
pyasn1==0.4.5
pycparser==2.19
PyPDF2==1.26.0
pyrsistent==0.14.9
python-dateutil==2.7.5
python-editor==1.0.3
python-json-logger==0.1.8
python-json-logger==0.1.10
pytz==2018.9
PyYAML==3.12
redis==3.0.1
requests==2.21.0
rsa==3.4.2
s3transfer==0.1.13
six==1.12.0
smartypants==2.0.1
statsd==3.2.2
statsd==3.3.0
urllib3==1.24.1
webencodings==0.5.1
Werkzeug==0.14.1

View File

@@ -1,235 +0,0 @@
import uuid
from datetime import datetime, timedelta
from freezegun import freeze_time
from app.dao.date_util import get_financial_year
from app.dao.notification_usage_dao import (
get_rates_for_daterange,
get_billing_data_for_month,
get_monthly_billing_data,
billing_letter_data_per_month_query
)
from app.models import (
Rate,
SMS_TYPE,
)
from tests.app.db import create_notification, create_rate, create_template, create_service
def test_get_rates_for_daterange(notify_db, notify_db_session):
set_up_rate(notify_db, datetime(2016, 5, 18), 0.016)
set_up_rate(notify_db, datetime(2017, 3, 31, 23), 0.0158)
start_date, end_date = get_financial_year(2017)
rates = get_rates_for_daterange(start_date, end_date, SMS_TYPE)
assert len(rates) == 1
assert datetime.strftime(rates[0].valid_from, '%Y-%m-%d %H:%M:%S') == "2017-03-31 23:00:00"
assert rates[0].rate == 0.0158
def test_get_rates_for_daterange_multiple_result_per_year(notify_db, notify_db_session):
set_up_rate(notify_db, datetime(2016, 4, 1), 0.015)
set_up_rate(notify_db, datetime(2016, 5, 18), 0.016)
set_up_rate(notify_db, datetime(2017, 4, 1), 0.0158)
start_date, end_date = get_financial_year(2016)
rates = get_rates_for_daterange(start_date, end_date, SMS_TYPE)
assert len(rates) == 2
assert datetime.strftime(rates[0].valid_from, '%Y-%m-%d %H:%M:%S') == "2016-04-01 00:00:00"
assert rates[0].rate == 0.015
assert datetime.strftime(rates[1].valid_from, '%Y-%m-%d %H:%M:%S') == "2016-05-18 00:00:00"
assert rates[1].rate == 0.016
def test_get_rates_for_daterange_returns_correct_rates(notify_db, notify_db_session):
set_up_rate(notify_db, datetime(2016, 4, 1), 0.015)
set_up_rate(notify_db, datetime(2016, 9, 1), 0.016)
set_up_rate(notify_db, datetime(2017, 6, 1), 0.0175)
start_date, end_date = get_financial_year(2017)
rates_2017 = get_rates_for_daterange(start_date, end_date, SMS_TYPE)
assert len(rates_2017) == 2
assert datetime.strftime(rates_2017[0].valid_from, '%Y-%m-%d %H:%M:%S') == "2016-09-01 00:00:00"
assert rates_2017[0].rate == 0.016
assert datetime.strftime(rates_2017[1].valid_from, '%Y-%m-%d %H:%M:%S') == "2017-06-01 00:00:00"
assert rates_2017[1].rate == 0.0175
def test_get_rates_for_daterange_in_the_future(notify_db, notify_db_session):
set_up_rate(notify_db, datetime(2016, 4, 1), 0.015)
set_up_rate(notify_db, datetime(2017, 6, 1), 0.0175)
start_date, end_date = get_financial_year(2018)
rates = get_rates_for_daterange(start_date, end_date, SMS_TYPE)
assert datetime.strftime(rates[0].valid_from, '%Y-%m-%d %H:%M:%S') == "2017-06-01 00:00:00"
assert rates[0].rate == 0.0175
def test_get_rates_for_daterange_returns_empty_list_if_year_is_before_earliest_rate(notify_db, notify_db_session):
set_up_rate(notify_db, datetime(2016, 4, 1), 0.015)
set_up_rate(notify_db, datetime(2017, 6, 1), 0.0175)
start_date, end_date = get_financial_year(2015)
rates = get_rates_for_daterange(start_date, end_date, SMS_TYPE)
assert rates == []
def test_get_rates_for_daterange_early_rate(notify_db, notify_db_session):
set_up_rate(notify_db, datetime(2015, 6, 1), 0.014)
set_up_rate(notify_db, datetime(2016, 6, 1), 0.015)
set_up_rate(notify_db, datetime(2016, 9, 1), 0.016)
set_up_rate(notify_db, datetime(2017, 6, 1), 0.0175)
start_date, end_date = get_financial_year(2016)
rates = get_rates_for_daterange(start_date, end_date, SMS_TYPE)
assert len(rates) == 3
def test_get_rates_for_daterange_edge_case(notify_db, notify_db_session):
set_up_rate(notify_db, datetime(2016, 3, 31, 23, 00), 0.015)
set_up_rate(notify_db, datetime(2017, 3, 31, 23, 00), 0.0175)
start_date, end_date = get_financial_year(2016)
rates = get_rates_for_daterange(start_date, end_date, SMS_TYPE)
assert len(rates) == 1
assert datetime.strftime(rates[0].valid_from, '%Y-%m-%d %H:%M:%S') == "2016-03-31 23:00:00"
assert rates[0].rate == 0.015
def test_get_rates_for_daterange_where_daterange_is_one_month_that_falls_between_rate_valid_from(
notify_db, notify_db_session
):
set_up_rate(notify_db, datetime(2017, 1, 1), 0.175)
set_up_rate(notify_db, datetime(2017, 3, 31), 0.123)
start_date = datetime(2017, 2, 1, 00, 00, 00)
end_date = datetime(2017, 2, 28, 23, 59, 59, 99999)
rates = get_rates_for_daterange(start_date, end_date, SMS_TYPE)
assert len(rates) == 1
assert datetime.strftime(rates[0].valid_from, '%Y-%m-%d %H:%M:%S') == "2017-01-01 00:00:00"
assert rates[0].rate == 0.175
def test_get_monthly_billing_data(notify_db, notify_db_session, sample_template, sample_email_template):
set_up_rate(notify_db, datetime(2016, 4, 1), 0.014)
# previous year
create_notification(template=sample_template, created_at=datetime(2016, 3, 31), sent_at=datetime(2016, 3, 31),
status='sending', billable_units=1)
# current year
create_notification(template=sample_template, created_at=datetime(2016, 4, 2), sent_at=datetime(2016, 4, 2),
status='sending', billable_units=1)
create_notification(template=sample_template, created_at=datetime(2016, 5, 18), sent_at=datetime(2016, 5, 18),
status='sending', billable_units=2)
create_notification(template=sample_template, created_at=datetime(2016, 7, 22), sent_at=datetime(2016, 7, 22),
status='sending', billable_units=3)
create_notification(template=sample_template, created_at=datetime(2016, 7, 22), sent_at=datetime(2016, 7, 22),
status='sending', billable_units=3, rate_multiplier=2)
create_notification(template=sample_template, created_at=datetime(2016, 7, 22), sent_at=datetime(2016, 7, 22),
status='sending', billable_units=3, rate_multiplier=2)
create_notification(template=sample_template, created_at=datetime(2016, 7, 30), sent_at=datetime(2016, 7, 22),
status='sending', billable_units=4)
create_notification(template=sample_email_template, created_at=datetime(2016, 8, 22), sent_at=datetime(2016, 7, 22),
status='sending', billable_units=0)
create_notification(template=sample_email_template, created_at=datetime(2016, 8, 30), sent_at=datetime(2016, 7, 22),
status='sending', billable_units=0)
# next year
create_notification(template=sample_template, created_at=datetime(2017, 3, 31, 23, 00, 00),
sent_at=datetime(2017, 3, 31), status='sending', billable_units=6)
results = get_monthly_billing_data(sample_template.service_id, 2016)
assert len(results) == 4
# (billable_units, rate_multiplier, international, type, rate)
assert results[0] == ('April', 1, 1, False, SMS_TYPE, 0.014)
assert results[1] == ('May', 2, 1, False, SMS_TYPE, 0.014)
assert results[2] == ('July', 7, 1, False, SMS_TYPE, 0.014)
assert results[3] == ('July', 6, 2, False, SMS_TYPE, 0.014)
def test_get_monthly_billing_data_with_multiple_rates(notify_db, notify_db_session, sample_template,
sample_email_template):
set_up_rate(notify_db, datetime(2016, 4, 1), 0.014)
set_up_rate(notify_db, datetime(2016, 6, 5), 0.0175)
set_up_rate(notify_db, datetime(2017, 7, 5), 0.018)
# previous year
create_notification(template=sample_template, created_at=datetime(2016, 3, 31), sent_at=datetime(2016, 3, 31),
status='sending', billable_units=1)
# current year
create_notification(template=sample_template, created_at=datetime(2016, 4, 2), sent_at=datetime(2016, 4, 2),
status='sending', billable_units=1)
create_notification(template=sample_template, created_at=datetime(2016, 5, 18), sent_at=datetime(2016, 5, 18),
status='sending', billable_units=2)
create_notification(template=sample_template, created_at=datetime(2016, 6, 1), sent_at=datetime(2016, 6, 1),
status='sending', billable_units=3)
create_notification(template=sample_template, created_at=datetime(2016, 6, 15), sent_at=datetime(2016, 6, 15),
status='sending', billable_units=4)
create_notification(template=sample_email_template, created_at=datetime(2016, 8, 22),
sent_at=datetime(2016, 7, 22),
status='sending', billable_units=0)
create_notification(template=sample_email_template, created_at=datetime(2016, 8, 30),
sent_at=datetime(2016, 7, 22),
status='sending', billable_units=0)
# next year
create_notification(template=sample_template, created_at=datetime(2017, 3, 31, 23, 00, 00),
sent_at=datetime(2017, 3, 31), status='sending', billable_units=6)
results = get_monthly_billing_data(sample_template.service_id, 2016)
assert len(results) == 4
assert results[0] == ('April', 1, 1, False, SMS_TYPE, 0.014)
assert results[1] == ('May', 2, 1, False, SMS_TYPE, 0.014)
assert results[2] == ('June', 3, 1, False, SMS_TYPE, 0.014)
assert results[3] == ('June', 4, 1, False, SMS_TYPE, 0.0175)
def test_get_monthly_billing_data_with_no_notifications_for_daterange(notify_db, notify_db_session, sample_template):
set_up_rate(notify_db, datetime(2016, 4, 1), 0.014)
results = get_monthly_billing_data(sample_template.service_id, 2016)
assert results == []
def set_up_rate(notify_db, start_date, value):
rate = Rate(id=uuid.uuid4(), valid_from=start_date, rate=value, notification_type=SMS_TYPE)
notify_db.session.add(rate)
@freeze_time("2016-05-01")
def test_get_billing_data_for_month_where_start_date_before_rate_returns_empty(
sample_template
):
create_rate(datetime(2016, 4, 1), 0.014, SMS_TYPE)
results = get_monthly_billing_data(
service_id=sample_template.service_id,
year=2015
)
assert not results
@freeze_time("2016-05-01")
def test_get_monthly_billing_data_where_start_date_before_rate_returns_empty(
sample_template
):
now = datetime.utcnow()
create_rate(now, 0.014, SMS_TYPE)
results = get_billing_data_for_month(
service_id=sample_template.service_id,
start_date=now - timedelta(days=2),
end_date=now - timedelta(days=1),
notification_type=SMS_TYPE
)
assert not results
def test_billing_letter_data_per_month_query(
notify_db_session
):
service = create_service()
template = create_template(service=service, template_type='letter')
create_notification(template=template, billable_units=1, created_at=datetime(2017, 2, 1, 13, 21),
status='delivered')
create_notification(template=template, billable_units=1, created_at=datetime(2017, 2, 1, 13, 21),
status='delivered')
create_notification(template=template, billable_units=1, created_at=datetime(2017, 2, 1, 13, 21),
status='delivered')
results = billing_letter_data_per_month_query(service_id=service.id,
start_date=datetime(2017, 2, 1),
end_date=datetime(2017, 2, 28))
assert len(results) == 1
assert results[0].rate == 0.3
assert results[0].billing_units == 3

View File

@@ -39,7 +39,7 @@ NewStatsRow = collections.namedtuple('row', ('notification_type', 'status', 'key
StatsRow('letter', 'virus-scan-failed', 1),
StatsRow('letter', 'permanent-failure', 1),
StatsRow('letter', 'cancelled', 1),
], [4, 0, 4], [0, 0, 0], [4, 0, 4]),
], [4, 0, 4], [0, 0, 0], [3, 0, 3]),
'convert_sent_to_delivered': ([
StatsRow('sms', 'sending', 1),
StatsRow('sms', 'delivered', 1),