From 1a64509186410c40f6da9daaea3fa1e56b948761 Mon Sep 17 00:00:00 2001 From: Rebecca Law Date: Thu, 27 Apr 2017 15:43:57 +0100 Subject: [PATCH] Change the resultset from the yearly and monthly billing data queries. Fix some formatting of the return objects. --- app/dao/notification_usage_dao.py | 119 +++++++++---------- app/models.py | 4 +- app/service/rest.py | 17 ++- tests/app/dao/test_notification_usage_dao.py | 44 ++++--- tests/app/db.py | 8 +- tests/app/service/test_rest.py | 37 ++++-- 6 files changed, 121 insertions(+), 108 deletions(-) diff --git a/app/dao/notification_usage_dao.py b/app/dao/notification_usage_dao.py index e07f575e0..fb412d849 100644 --- a/app/dao/notification_usage_dao.py +++ b/app/dao/notification_usage_dao.py @@ -1,5 +1,8 @@ from datetime import datetime -from sqlalchemy import func + +from sqlalchemy import Float, Integer +from sqlalchemy import func, case, cast +from sqlalchemy import literal_column from app import db from app.dao.date_util import get_financial_year @@ -21,13 +24,26 @@ def get_yearly_billing_data(service_id, year): result = [] for r, n in zip(rates, rates[1:]): result.append( - sms_billing_data_query(r.rate, service_id, r.valid_from, n.valid_from)) + sms_yearly_billing_data_query(r.rate, service_id, r.valid_from, n.valid_from)) - result.append(sms_billing_data_query(rates[-1].rate, service_id, rates[-1].valid_from, end_date)) + result.append(sms_yearly_billing_data_query(rates[-1].rate, service_id, rates[-1].valid_from, end_date)) - result.append(email_billing_data_query(service_id, start_date, end_date)) + result.append(email_yearly_billing_data_query(service_id, start_date, end_date)) - return result + return sum(result, []) + + +@statsd(namespace="dao") +def get_monthly_billing_data(service_id, year): + start_date, end_date = get_financial_year(year) + rates = get_rates_for_year(start_date, end_date, SMS_TYPE) + + result = [] + for r, n in zip(rates, rates[1:]): + result.extend(sms_billing_data_per_month_query(r.rate, service_id, r.valid_from, n.valid_from)) + result.extend(sms_billing_data_per_month_query(rates[-1].rate, service_id, rates[-1].valid_from, end_date)) + + return [(datetime.strftime(x[0], "%B"), x[1], x[2], x[3], x[4], x[5]) for x in result] def billing_data_filter(notification_type, start_date, end_date, service_id): @@ -41,46 +57,49 @@ def billing_data_filter(notification_type, start_date, end_date, service_id): ] -def email_billing_data_query(service_id, start_date, end_date): +def email_yearly_billing_data_query(service_id, start_date, end_date, rate=0): result = db.session.query( func.count(NotificationHistory.id), + func.count(NotificationHistory.id), + rate_multiplier(), NotificationHistory.notification_type, - NotificationHistory.rate_multiplier, NotificationHistory.international, - NotificationHistory.phone_prefix + cast(rate, Integer()) ).filter( *billing_data_filter(EMAIL_TYPE, start_date, end_date, service_id) ).group_by( NotificationHistory.notification_type, - NotificationHistory.rate_multiplier, - NotificationHistory.international, - NotificationHistory.phone_prefix + rate_multiplier(), + NotificationHistory.international ).first() if not result: - return 0, EMAIL_TYPE, None, False, None, 0 + return [(0, 0, 1, EMAIL_TYPE, False, 0)] else: - return tuple(result) + (0,) + return [result] -def sms_billing_data_query(rate, service_id, start_date, end_date): +def sms_yearly_billing_data_query(rate, service_id, start_date, end_date): result = db.session.query( + cast(func.sum(NotificationHistory.billable_units * rate_multiplier()), Integer()), func.sum(NotificationHistory.billable_units), + rate_multiplier(), NotificationHistory.notification_type, - NotificationHistory.rate_multiplier, NotificationHistory.international, - NotificationHistory.phone_prefix + cast(rate, Float()) ).filter( *billing_data_filter(SMS_TYPE, start_date, end_date, service_id) ).group_by( NotificationHistory.notification_type, - NotificationHistory.rate_multiplier, NotificationHistory.international, - NotificationHistory.phone_prefix - ).first() + rate_multiplier() + ).order_by( + rate_multiplier() + ).all() + if not result: - return 0, SMS_TYPE, None, False, None, 0 + return [(0, 0, 1, SMS_TYPE, False, rate)] else: - return tuple(result) + (rate,) + return result def get_rates_for_year(start_date, end_date, notification_type): @@ -90,58 +109,30 @@ def get_rates_for_year(start_date, end_date, notification_type): def sms_billing_data_per_month_query(rate, service_id, start_date, end_date): month = get_london_month_from_utc_column(NotificationHistory.created_at) - return [result + (rate,) for result in db.session.query( + result = db.session.query( month, func.sum(NotificationHistory.billable_units), - NotificationHistory.notification_type, - NotificationHistory.rate_multiplier, + rate_multiplier(), NotificationHistory.international, - NotificationHistory.phone_prefix + NotificationHistory.notification_type, + cast(rate, Float()) ).filter( *billing_data_filter(SMS_TYPE, start_date, end_date, service_id) ).group_by( - NotificationHistory.notification_type, month, - NotificationHistory.rate_multiplier, - NotificationHistory.international, - NotificationHistory.phone_prefix - ).order_by( - month - ).all() - ] - - -def email_billing_data_per_month_query(rate, service_id, start_date, end_date): - month = get_london_month_from_utc_column(NotificationHistory.created_at) - return [result + (rate,) for result in db.session.query( - month, - func.count(NotificationHistory.id), NotificationHistory.notification_type, + month, NotificationHistory.rate_multiplier, - NotificationHistory.international, - NotificationHistory.phone_prefix - ).filter( - *billing_data_filter(EMAIL_TYPE, start_date, end_date, service_id) - ).group_by( - NotificationHistory.notification_type, month, - NotificationHistory.rate_multiplier, - NotificationHistory.international, - NotificationHistory.phone_prefix + NotificationHistory.international ).order_by( - month + month, + rate_multiplier() ).all() - ] + + return result -@statsd(namespace="dao") -def get_monthly_billing_data(service_id, year): - start_date, end_date = get_financial_year(year) - rates = get_rates_for_year(start_date, end_date, SMS_TYPE) - - result = [] - for r, n in zip(rates, rates[1:]): - result.extend(sms_billing_data_per_month_query(r.rate, service_id, r.valid_from, n.valid_from)) - result.extend(sms_billing_data_per_month_query(rates[-1].rate, service_id, rates[-1].valid_from, end_date)) - - result.extend(email_billing_data_per_month_query(0, service_id, start_date, end_date)) - - return [(datetime.strftime(x[0], "%B"), x[1], x[2], x[3], x[4], x[5], x[6]) for x in result] +def rate_multiplier(): + return cast(case([ + (NotificationHistory.rate_multiplier == None, literal_column("'1'")), # noqa + (NotificationHistory.rate_multiplier != None, NotificationHistory.rate_multiplier), # noqa + ]), Integer()) diff --git a/app/models.py b/app/models.py index b6af23aac..59e6f7a91 100644 --- a/app/models.py +++ b/app/models.py @@ -847,7 +847,7 @@ class NotificationHistory(db.Model, HistoryModel): international = db.Column(db.Boolean, nullable=False, default=False) phone_prefix = db.Column(db.String, nullable=True) - rate_multiplier = db.Column(db.Float(), nullable=True) + rate_multiplier = db.Column(db.Float(asdecimal=False), nullable=True) @classmethod def from_original(cls, notification): @@ -968,5 +968,5 @@ class Rate(db.Model): id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) valid_from = db.Column(db.DateTime, nullable=False) - rate = db.Column(db.Numeric(asdecimal=False), nullable=False) + rate = db.Column(db.Float(asdecimal=False), nullable=False) notification_type = db.Column(notification_types, index=True, nullable=False) diff --git a/app/service/rest.py b/app/service/rest.py index 8027125bd..25991f30d 100644 --- a/app/service/rest.py +++ b/app/service/rest.py @@ -420,11 +420,11 @@ def get_yearly_billing_usage(service_id): try: year = int(request.args.get('year')) results = notification_usage_dao.get_yearly_billing_data(service_id, year) - json_result = [{"billing_units": x[0], - "notification_type": x[1], + json_result = [{"credits": x[0], + "billing_units": x[1], "rate_multiplier": x[2], - "international": x[3], - "phone_prefix": x[4], + "notification_type": x[3], + "international": x[4], "rate": x[5] } for x in results] return json.dumps(json_result) @@ -440,11 +440,10 @@ def get_yearly_monthly_usage(service_id): results = notification_usage_dao.get_monthly_billing_data(service_id, year) json_results = [{"month": x[0], "billing_units": x[1], - "notification_type": x[2], - "rate_multiplier": x[3], - "international": x[4], - "phone_prefix": x[5], - "rate": x[6] + "rate_multiplier": x[2], + "international": x[3], + "notification_type": x[4], + "rate": x[5] } for x in results] return json.dumps(json_results) except TypeError: diff --git a/tests/app/dao/test_notification_usage_dao.py b/tests/app/dao/test_notification_usage_dao.py index 75942f15a..72891d46e 100644 --- a/tests/app/dao/test_notification_usage_dao.py +++ b/tests/app/dao/test_notification_usage_dao.py @@ -32,7 +32,7 @@ def test_get_yearly_billing_data(notify_db, notify_db_session, sample_template, create_notification(template=sample_template, created_at=datetime(2016, 5, 18), sent_at=datetime(2016, 5, 18), status='sending', billable_units=2) create_notification(template=sample_template, created_at=datetime(2016, 7, 22), sent_at=datetime(2016, 7, 22), - status='sending', billable_units=3) + status='sending', billable_units=3, rate_multiplier=2, international=True, phone_prefix="1") create_notification(template=sample_template, created_at=datetime(2016, 9, 15), sent_at=datetime(2016, 9, 15), status='sending', billable_units=4) create_notification(template=sample_template, created_at=datetime(2017, 3, 31), sent_at=datetime(2017, 3, 31), @@ -45,10 +45,11 @@ def test_get_yearly_billing_data(notify_db, notify_db_session, sample_template, create_notification(template=sample_template, created_at=datetime(2017, 4, 1), sent_at=datetime(2017, 4, 1), status='sending', billable_units=6) results = get_yearly_billing_data(sample_template.service_id, 2016) - assert len(results) == 3 - assert results[0] == (3, 'sms', None, False, None, 1.40) - assert results[1] == (12, 'sms', None, False, None, 1.58) - assert results[2] == (2, 'email', None, False, None, 0) + assert len(results) == 4 + assert results[0] == (3, 3, 1, 'sms', False, 1.4) + assert results[1] == (9, 9, 1, 'sms', False, 1.58) + assert results[2] == (6, 3, 2, 'sms', True, 1.58) + assert results[3] == (2, 2, 1, 'email', False, 0) def test_get_yearly_billing_data_with_one_rate(notify_db, notify_db_session, sample_template): @@ -74,8 +75,8 @@ def test_get_yearly_billing_data_with_one_rate(notify_db, notify_db_session, sam status='sending', billable_units=7) results = get_yearly_billing_data(sample_template.service_id, 2016) assert len(results) == 2 - assert results[0] == (15, 'sms', None, False, None, 1.4) - assert results[1] == (0, 'email', None, False, None, 0) + assert results[0] == (15, 15, 1, 'sms', False, 1.4) + assert results[1] == (0, 0, 1, 'email', False, 0) def test_get_yearly_billing_data_with_no_sms_notifications(notify_db, notify_db_session, sample_email_template): @@ -87,8 +88,8 @@ def test_get_yearly_billing_data_with_no_sms_notifications(notify_db, notify_db_ results = get_yearly_billing_data(sample_email_template.service_id, 2016) assert len(results) == 2 - assert results[0] == (0, 'sms', None, False, None, 0) - assert results[1] == (2, 'email', None, False, None, 0) + assert results[0] == (0, 0, 1, 'sms', False, 1.4) + assert results[1] == (2, 2, 1, 'email', False, 0) def test_get_monthly_billing_data(notify_db, notify_db_session, sample_template, sample_email_template): @@ -103,8 +104,13 @@ def test_get_monthly_billing_data(notify_db, notify_db_session, sample_template, status='sending', billable_units=2) create_notification(template=sample_template, created_at=datetime(2016, 7, 22), sent_at=datetime(2016, 7, 22), status='sending', billable_units=3) + create_notification(template=sample_template, created_at=datetime(2016, 7, 22), sent_at=datetime(2016, 7, 22), + status='sending', billable_units=3, rate_multiplier=2) + create_notification(template=sample_template, created_at=datetime(2016, 7, 22), sent_at=datetime(2016, 7, 22), + status='sending', billable_units=3, rate_multiplier=2) create_notification(template=sample_template, created_at=datetime(2016, 7, 30), sent_at=datetime(2016, 7, 22), status='sending', billable_units=4) + create_notification(template=sample_email_template, created_at=datetime(2016, 8, 22), sent_at=datetime(2016, 7, 22), status='sending', billable_units=0) create_notification(template=sample_email_template, created_at=datetime(2016, 8, 30), sent_at=datetime(2016, 7, 22), @@ -114,10 +120,11 @@ def test_get_monthly_billing_data(notify_db, notify_db_session, sample_template, sent_at=datetime(2017, 3, 31), status='sending', billable_units=6) results = get_monthly_billing_data(sample_template.service_id, 2016) assert len(results) == 4 - assert results[0] == ('April', 1, 'sms', None, False, None, 1.4) - assert results[1] == ('May', 2, 'sms', None, False, None, 1.4) - assert results[2] == ('July', 7, 'sms', None, False, None, 1.4) - assert results[3] == ('August', 2, 'email', None, False, None, 0) + # (billable_units, rate_multiplier, international, type, rate) + assert results[0] == ('April', 1, 1, False, 'sms', 1.4) + assert results[1] == ('May', 2, 1, False, 'sms', 1.4) + assert results[2] == ('July', 7, 1, False, 'sms', 1.4) + assert results[3] == ('July', 6, 2, False, 'sms', 1.4) def test_get_monthly_billing_data_with_multiple_rates(notify_db, notify_db_session, sample_template, @@ -146,12 +153,11 @@ def test_get_monthly_billing_data_with_multiple_rates(notify_db, notify_db_sessi create_notification(template=sample_template, created_at=datetime(2017, 3, 31, 23, 00, 00), sent_at=datetime(2017, 3, 31), status='sending', billable_units=6) results = get_monthly_billing_data(sample_template.service_id, 2016) - assert len(results) == 5 - assert results[0] == ('April', 1, 'sms', None, False, None, 1.4) - assert results[1] == ('May', 2, 'sms', None, False, None, 1.4) - assert results[2] == ('June', 3, 'sms', None, False, None, 1.4) - assert results[3] == ('June', 4, 'sms', None, False, None, 1.75) - assert results[4] == ('August', 2, 'email', None, False, None, 0) + assert len(results) == 4 + assert results[0] == ('April', 1, 1, False, 'sms', 1.4) + assert results[1] == ('May', 2, 1, False, 'sms', 1.4) + assert results[2] == ('June', 3, 1, False, 'sms', 1.4) + assert results[3] == ('June', 4, 1, False, 'sms', 1.75) def set_up_rate(notify_db, start_date, value): diff --git a/tests/app/db.py b/tests/app/db.py index 38d5eca9d..b5838d693 100644 --- a/tests/app/db.py +++ b/tests/app/db.py @@ -73,7 +73,9 @@ def create_notification( key_type=KEY_TYPE_NORMAL, sent_by=None, client_reference=None, - international=False + rate_multiplier=None, + international=False, + phone_prefix=None ): if created_at is None: created_at = datetime.utcnow() @@ -105,7 +107,9 @@ def create_notification( 'updated_at': updated_at, 'client_reference': client_reference, 'job_row_number': job_row_number, - 'international': international + 'rate_multiplier': rate_multiplier, + 'international': international, + 'phone_prefix': phone_prefix } notification = Notification(**data) dao_create_notification(notification) diff --git a/tests/app/service/test_rest.py b/tests/app/service/test_rest.py index 37331b182..89cfc27f1 100644 --- a/tests/app/service/test_rest.py +++ b/tests/app/service/test_rest.py @@ -1523,12 +1523,18 @@ def test_get_yearly_billing_usage(client, notify_db, notify_db_session): ) assert response.status_code == 200 - assert json.loads(response.get_data(as_text=True)) == [{'billing_units': 1, 'notification_type': 'sms', - 'phone_prefix': None, 'international': False, - 'rate_multiplier': None, 'rate': 1.58}, - {'billing_units': 0, 'notification_type': 'email', - 'phone_prefix': None, 'international': False, - 'rate_multiplier': None, 'rate': 0}] + assert json.loads(response.get_data(as_text=True)) == [{'credits': 1, + 'billing_units': 1, + 'rate_multiplier': 1, + 'notification_type': 'sms', + 'international': False, + 'rate': 1.58}, + {'credits': 0, + 'billing_units': 0, + 'rate_multiplier': 1, + 'notification_type': 'email', + 'international': False, + 'rate': 0}] def test_get_yearly_billing_usage_returns_400_if_missing_year(client, sample_service): @@ -1558,12 +1564,19 @@ def test_get_monthly_billing_usage(client, notify_db, notify_db_session): assert response.status_code == 200 actual = json.loads(response.get_data(as_text=True)) assert len(actual) == 2 - assert actual == [{'month': 'June', 'billing_units': 1, 'notification_type': 'sms', - 'phone_prefix': None, 'international': False, - 'rate_multiplier': None, 'rate': 1.58}, - {'month': 'July', 'billing_units': 1, 'notification_type': 'sms', - 'phone_prefix': None, 'international': False, - 'rate_multiplier': None, 'rate': 1.58}] + print(actual) + assert actual == [{'month': 'June', + 'international': False, + 'rate_multiplier': 1, + 'notification_type': 'sms', + 'rate': 1.58, + 'billing_units': 1}, + {'month': 'July', + 'international': False, + 'rate_multiplier': 1, + 'notification_type': 'sms', + 'rate': 1.58, + 'billing_units': 1}] def test_get_monthly_billing_usage_returns_400_if_missing_year(client, sample_service):