From 4b05c32b622d36ee8e5f3b1b9ca3d99735bc2c1f Mon Sep 17 00:00:00 2001 From: Rebecca Law Date: Thu, 13 Jul 2017 17:22:11 +0100 Subject: [PATCH 1/6] Create a new table to warehouse the monthly billing numbers --- app/dao/monthly_billing_dao.py | 6 ++++ app/models.py | 17 +++++++++- migrations/versions/0109_monthly_billing.py | 37 +++++++++++++++++++++ tests/app/dao/test_monthly_billing.py | 33 ++++++++++++++++++ 4 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 app/dao/monthly_billing_dao.py create mode 100644 migrations/versions/0109_monthly_billing.py create mode 100644 tests/app/dao/test_monthly_billing.py diff --git a/app/dao/monthly_billing_dao.py b/app/dao/monthly_billing_dao.py new file mode 100644 index 000000000..c037b5c54 --- /dev/null +++ b/app/dao/monthly_billing_dao.py @@ -0,0 +1,6 @@ +from app import db + + +def update_monthly_billing(monthly_billing): + db.session.add(monthly_billing) + db.session.commit() diff --git a/app/models.py b/app/models.py index 0e3079c80..07ba505d8 100644 --- a/app/models.py +++ b/app/models.py @@ -4,7 +4,6 @@ import datetime from flask import url_for, current_app from sqlalchemy.ext.associationproxy import association_proxy -from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.dialects.postgresql import ( UUID, JSON @@ -1246,3 +1245,19 @@ class LetterRateDetail(db.Model): letter_rate = db.relationship('LetterRate', backref='letter_rates') page_total = db.Column(db.Integer, nullable=False) rate = db.Column(db.Numeric(), nullable=False) + + +class MonthlyBilling(db.Model): + __tablename__ = 'monthly_billing' + + id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'), index=True, nullable=False) + service = db.relationship('Service', backref='monthly_billing') + month = db.Column(db.String, nullable=False) + year = db.Column(db.Float(asdecimal=False), nullable=False) + notification_type = db.Column(notification_types, nullable=False) + monthly_totals = db.Column(JSON, nullable=False) + + __table_args__ = ( + UniqueConstraint('service_id', 'month', 'year', 'notification_type', name='uix_monthly_billing'), + ) diff --git a/migrations/versions/0109_monthly_billing.py b/migrations/versions/0109_monthly_billing.py new file mode 100644 index 000000000..124a509cf --- /dev/null +++ b/migrations/versions/0109_monthly_billing.py @@ -0,0 +1,37 @@ +"""empty message + +Revision ID: 0109_monthly_billing +Revises: 0108_change_logo_not_nullable +Create Date: 2017-07-13 14:35:03.183659 + +""" + +# revision identifiers, used by Alembic. +revision = '0109_monthly_billing' +down_revision = '0108_change_logo_not_nullable' + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + + +def upgrade(): + + op.create_table('monthly_billing', + sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('service_id', postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('month', sa.String(), nullable=False), + sa.Column('year', sa.Float(), nullable=False), + sa.Column('notification_type', + postgresql.ENUM('email', 'sms', 'letter', name='notification_type', create_type=False), + nullable=False), + sa.Column('monthly_totals', postgresql.JSON(), nullable=False), + sa.ForeignKeyConstraint(['service_id'], ['services.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_monthly_billing_service_id'), 'monthly_billing', ['service_id'], unique=False) + op.create_index(op.f('uix_monthly_billing'), 'monthly_billing', ['service_id', 'month', 'year', 'notification_type'], unique=True) + + +def downgrade(): + op.drop_table('monthly_billing') diff --git a/tests/app/dao/test_monthly_billing.py b/tests/app/dao/test_monthly_billing.py new file mode 100644 index 000000000..f04fa79ca --- /dev/null +++ b/tests/app/dao/test_monthly_billing.py @@ -0,0 +1,33 @@ +import uuid + +import pytest +from sqlalchemy.exc import IntegrityError + +from app.dao.monthly_billing_dao import update_monthly_billing +from app.models import MonthlyBilling + + +def test_add_monthly_billing_only_allows_one_row_per_service_month_type(sample_service): + first = MonthlyBilling(id=uuid.uuid4(), + service_id=sample_service.id, + notification_type='sms', + month='January', + year='2017', + monthly_totals={'billing_units': 100, + 'rate': 0.0158}) + + second = MonthlyBilling(id=uuid.uuid4(), + service_id=sample_service.id, + notification_type='sms', + month='January', + year='2017', + monthly_totals={'billing_units': 50, + 'rate': 0.0162}) + + update_monthly_billing(first) + with pytest.raises(IntegrityError): + update_monthly_billing(second) + monthly = MonthlyBilling.query.all() + assert len(monthly) == 1 + assert monthly[0].monthly_totals == {'billing_units': 100, + 'rate': 0.0158} From 9400988d72e97f9613f4681293f54018a3650a5b Mon Sep 17 00:00:00 2001 From: Rebecca Law Date: Tue, 18 Jul 2017 18:21:35 +0100 Subject: [PATCH 2/6] Monthly billing - part 1 This is still a work in progress but it would be good to get some eyes on it. This commit includes creating and updating a row in the monthly billing table and a method to fetch the results. There is a command to populate the monthly billing for a service and month so we can try it out. The total cost at the moment are wrong, they do not take into account the free allowance - see notes below about adding that to the table. Left to do: create a nightly task to run to update the monthly totals. create an endpoint to return the yearly billing, the current day will need to be calculated on the fly and added to the totals. Add the free allowance into the total costs. --- app/commands.py | 18 +++- app/dao/date_util.py | 13 +++ app/dao/monthly_billing_dao.py | 42 +++++++- app/dao/notification_usage_dao.py | 17 ++- app/dao/provider_rates_dao.py | 5 + app/delivery/send_to_providers.py | 2 +- app/models.py | 9 ++ app/service/rest.py | 1 - application.py | 1 + tests/app/dao/test_date_utils.py | 19 +++- tests/app/dao/test_monthly_billing.py | 128 ++++++++++++++++++----- tests/app/dao/test_provider_rates_dao.py | 13 ++- tests/app/db.py | 9 +- 13 files changed, 240 insertions(+), 37 deletions(-) diff --git a/app/commands.py b/app/commands.py index 6e4bc53d1..c7ec0567b 100644 --- a/app/commands.py +++ b/app/commands.py @@ -5,7 +5,8 @@ from flask.ext.script import Command, Manager, Option from app import db -from app.models import (PROVIDERS, Service, User, NotificationHistory) +from app.dao.monthly_billing_dao import create_or_update_monthly_billing_sms, get_monthly_billing_sms +from app.models import (PROVIDERS, User) from app.dao.services_dao import ( delete_service_and_all_associated_db_objects, dao_fetch_all_services_by_user @@ -146,3 +147,18 @@ class CustomDbScript(Command): print('Committed {} updates at {}'.format(len(result), datetime.utcnow())) db.session.commit() result = db.session.execute(subq_hist).fetchall() + + +class PopulateMonthlyBilling(Command): + option_list = ( + Option('-s', '-service-id', dest='service_id', + help="Service id to populate monthly billing for"), + Option('-m', '-month', dest="month", help="Use for integer value for month, e.g. 7 for July"), + Option('-y', '-year', dest="year", help="Use for integer value for year, e.g. 2017") + ) + + def run(self, service_id, month, year): + create_or_update_monthly_billing_sms(service_id, datetime(int(year), int(month), 1)) + results = get_monthly_billing_sms(service_id, datetime(int(year), int(month), 1)) + print("Finished populating data for {} for service id {}".format(month, service_id)) + print(results.monthly_totals) diff --git a/app/dao/date_util.py b/app/dao/date_util.py index 2471865bd..41ad9dc72 100644 --- a/app/dao/date_util.py +++ b/app/dao/date_util.py @@ -16,3 +16,16 @@ def get_april_fools(year): """ return pytz.timezone('Europe/London').localize(datetime(year, 4, 1, 0, 0, 0)).astimezone(pytz.UTC).replace( tzinfo=None) + + +def get_month_start_end_date(month_year): + """ + This function return the start and date of the month_year as UTC, + :param month_year: the datetime to calculate the start and end date for that month + :return: start_date, end_date, month + """ + import calendar + _, num_days = calendar.monthrange(month_year.year, month_year.month) + first_day = datetime(month_year.year, month_year.month, 1, 0, 0, 0) + last_day = datetime(month_year.year, month_year.month, num_days, 23, 59, 59, 99999) + return first_day, last_day diff --git a/app/dao/monthly_billing_dao.py b/app/dao/monthly_billing_dao.py index c037b5c54..8e3204584 100644 --- a/app/dao/monthly_billing_dao.py +++ b/app/dao/monthly_billing_dao.py @@ -1,6 +1,44 @@ +from datetime import datetime + from app import db +from app.dao.notification_usage_dao import get_billing_data_for_month +from app.models import MonthlyBilling, SMS_TYPE -def update_monthly_billing(monthly_billing): - db.session.add(monthly_billing) +def create_or_update_monthly_billing_sms(service_id, billing_month): + monthly = get_billing_data_for_month(service_id=service_id, billing_month=billing_month) + # update monthly + monthly_totals = _monthly_billing_data_to_json(monthly) + row = MonthlyBilling.query.filter_by(year=billing_month.year, + month=datetime.strftime(billing_month, "%B"), + notification_type='sms').first() + if row: + row.monthly_totals = monthly_totals + else: + row = MonthlyBilling(service_id=service_id, + notification_type=SMS_TYPE, + year=billing_month.year, + month=datetime.strftime(billing_month, "%B"), + monthly_totals=monthly_totals) + db.session.add(row) db.session.commit() + + +def get_monthly_billing_sms(service_id, billing_month): + monthly = MonthlyBilling.query.filter_by(service_id=service_id, + year=billing_month.year, + month=datetime.strftime(billing_month, "%B"), + notification_type=SMS_TYPE).first() + return monthly + + +def _monthly_billing_data_to_json(monthly): + # ('April', 6, 1, False, 'sms', 0.014) + # (month, billing_units, rate_multiplier, international, notification_type, rate) + # total cost must take into account the free allowance. + # might be a good idea to capture free allowance in this table + return [{"billing_units": x[1], + "rate_multiplier": x[2], + "international": x[3], + "rate": x[5], + "total_cost": (x[1] * x[2]) * x[5]} for x in monthly] diff --git a/app/dao/notification_usage_dao.py b/app/dao/notification_usage_dao.py index 995462509..289a5a03d 100644 --- a/app/dao/notification_usage_dao.py +++ b/app/dao/notification_usage_dao.py @@ -6,7 +6,7 @@ from sqlalchemy import func, case, cast from sqlalchemy import literal_column from app import db -from app.dao.date_util import get_financial_year +from app.dao.date_util import get_financial_year, get_month_start_end_date from app.models import (NotificationHistory, Rate, NOTIFICATION_STATUS_TYPES_BILLABLE, @@ -35,6 +35,21 @@ def get_yearly_billing_data(service_id, year): return sum(result, []) +@statsd(namespace="dao") +def get_billing_data_for_month(service_id, billing_month): + start_date, end_date = get_month_start_end_date(billing_month) + rates = get_rates_for_year(start_date, end_date, SMS_TYPE) + result = [] + # so the start end date in the query are the valid from the rate, not the month - this is going to take some thought + for r, n in zip(rates, rates[1:]): + result.extend(sms_billing_data_per_month_query(r.rate, service_id, max(r.valid_from, start_date), + min(n.valid_from, end_date))) + result.extend( + sms_billing_data_per_month_query(rates[-1].rate, service_id, max(rates[-1].valid_from, start_date), end_date)) + + return result + + @statsd(namespace="dao") def get_monthly_billing_data(service_id, year): start_date, end_date = get_financial_year(year) diff --git a/app/dao/provider_rates_dao.py b/app/dao/provider_rates_dao.py index 145bd431e..443543f86 100644 --- a/app/dao/provider_rates_dao.py +++ b/app/dao/provider_rates_dao.py @@ -9,3 +9,8 @@ def create_provider_rates(provider_identifier, valid_from, rate): provider_rates = ProviderRates(provider_id=provider.id, valid_from=valid_from, rate=rate) db.session.add(provider_rates) + + +@transactional +def create_sms_rate(rate): + db.session.add(rate) diff --git a/app/delivery/send_to_providers.py b/app/delivery/send_to_providers.py index df4c664f9..5e33e2113 100644 --- a/app/delivery/send_to_providers.py +++ b/app/delivery/send_to_providers.py @@ -18,7 +18,7 @@ from app.dao.templates_dao import dao_get_template_by_id from app.models import SMS_TYPE, KEY_TYPE_TEST, BRANDING_ORG, EMAIL_TYPE, NOTIFICATION_TECHNICAL_FAILURE, \ NOTIFICATION_SENT, NOTIFICATION_SENDING -from app.celery.statistics_tasks import record_initial_job_statistics, create_initial_notification_statistic_tasks +from app.celery.statistics_tasks import create_initial_notification_statistic_tasks def send_sms_to_provider(notification): diff --git a/app/models.py b/app/models.py index 07ba505d8..e035acccb 100644 --- a/app/models.py +++ b/app/models.py @@ -1261,3 +1261,12 @@ class MonthlyBilling(db.Model): __table_args__ = ( UniqueConstraint('service_id', 'month', 'year', 'notification_type', name='uix_monthly_billing'), ) + + def serialized(self): + return { + "month": self.month, + "year": self.year, + "service_id": str(self.service_id), + "notification_type": self.notification_type, + "monthly_totals": self.monthly_totals + } diff --git a/app/service/rest.py b/app/service/rest.py index ae57f5390..deb37f525 100644 --- a/app/service/rest.py +++ b/app/service/rest.py @@ -68,7 +68,6 @@ from app.schemas import ( user_schema, permission_schema, notification_with_template_schema, - notification_with_personalisation_schema, notifications_filter_schema, detailed_service_schema ) diff --git a/application.py b/application.py index b2a59a6fd..5ff7dd1e4 100644 --- a/application.py +++ b/application.py @@ -16,6 +16,7 @@ manager.add_command('db', MigrateCommand) manager.add_command('create_provider_rate', commands.CreateProviderRateCommand) manager.add_command('purge_functional_test_data', commands.PurgeFunctionalTestDataCommand) manager.add_command('custom_db_script', commands.CustomDbScript) +manager.add_command('populate_monthly_billing', commands.PopulateMonthlyBilling) @manager.command diff --git a/tests/app/dao/test_date_utils.py b/tests/app/dao/test_date_utils.py index 760923c5a..4ede18853 100644 --- a/tests/app/dao/test_date_utils.py +++ b/tests/app/dao/test_date_utils.py @@ -1,4 +1,8 @@ -from app.dao.date_util import get_financial_year, get_april_fools +from datetime import datetime + +import pytest + +from app.dao.date_util import get_financial_year, get_april_fools, get_month_start_end_date def test_get_financial_year(): @@ -11,3 +15,16 @@ def test_get_april_fools(): april_fools = get_april_fools(2016) assert str(april_fools) == '2016-03-31 23:00:00' assert april_fools.tzinfo is None + + +@pytest.mark.parametrize("month, year, expected_end", + [(7, 2017, 31), + (2, 2016, 29), + (2, 2017, 28), + (9, 2018, 30), + (12, 2019, 31)]) +def test_get_month_start_end_date(month, year, expected_end): + month_year = datetime(year, month, 10, 13, 30, 00) + result = get_month_start_end_date(month_year) + assert result[0] == datetime(year, month, 1, 0, 0, 0, 0) + assert result[1] == datetime(year, month, expected_end, 23, 59, 59, 99999) diff --git a/tests/app/dao/test_monthly_billing.py b/tests/app/dao/test_monthly_billing.py index f04fa79ca..6a0399d67 100644 --- a/tests/app/dao/test_monthly_billing.py +++ b/tests/app/dao/test_monthly_billing.py @@ -1,33 +1,107 @@ -import uuid +from datetime import datetime -import pytest -from sqlalchemy.exc import IntegrityError - -from app.dao.monthly_billing_dao import update_monthly_billing +from app.dao.monthly_billing_dao import create_or_update_monthly_billing_sms, get_monthly_billing_sms from app.models import MonthlyBilling +from tests.app.db import create_notification, create_rate -def test_add_monthly_billing_only_allows_one_row_per_service_month_type(sample_service): - first = MonthlyBilling(id=uuid.uuid4(), - service_id=sample_service.id, - notification_type='sms', - month='January', - year='2017', - monthly_totals={'billing_units': 100, - 'rate': 0.0158}) +def test_add_monthly_billing(sample_template): + jan = datetime(2017, 1, 1) + feb = datetime(2017, 2, 15) + create_rate(start_date=jan, value=0.0158, notification_type='sms') + create_notification(template=sample_template, created_at=jan, billable_units=1, status='delivered') + create_notification(template=sample_template, created_at=feb, billable_units=2, status='delivered') - second = MonthlyBilling(id=uuid.uuid4(), - service_id=sample_service.id, - notification_type='sms', - month='January', - year='2017', - monthly_totals={'billing_units': 50, - 'rate': 0.0162}) + create_or_update_monthly_billing_sms(service_id=sample_template.service_id, + billing_month=jan) + create_or_update_monthly_billing_sms(service_id=sample_template.service_id, + billing_month=feb) + monthly_billing = MonthlyBilling.query.all() + assert len(monthly_billing) == 2 + assert monthly_billing[0].month == 'January' + assert monthly_billing[1].month == 'February' - update_monthly_billing(first) - with pytest.raises(IntegrityError): - update_monthly_billing(second) - monthly = MonthlyBilling.query.all() - assert len(monthly) == 1 - assert monthly[0].monthly_totals == {'billing_units': 100, - 'rate': 0.0158} + january = get_monthly_billing_sms(service_id=sample_template.service_id, billing_month=jan) + expected_jan = {"billing_units": 1, + "rate_multiplier": 1, + "international": False, + "rate": 0.0158, + "total_cost": 1 * 0.0158} + assert_monthly_billing(january, 2017, "January", sample_template.service_id, 1, expected_jan) + + february = get_monthly_billing_sms(service_id=sample_template.service_id, billing_month=feb) + expected_feb = {"billing_units": 2, + "rate_multiplier": 1, + "international": False, + "rate": 0.0158, + "total_cost": 2 * 0.0158} + assert_monthly_billing(february, 2017, "February", sample_template.service_id, 1, expected_feb) + + +def test_add_monthly_billing_multiple_rates_in_a_month(sample_template): + rate_1 = datetime(2016, 12, 1) + rate_2 = datetime(2017, 1, 15) + create_rate(start_date=rate_1, value=0.0158, notification_type='sms') + create_rate(start_date=rate_2, value=0.0124, notification_type='sms') + + create_notification(template=sample_template, created_at=datetime(2017, 1, 1), billable_units=1, status='delivered') + create_notification(template=sample_template, created_at=datetime(2017, 1, 14, 23, 59), billable_units=1, + status='delivered') + + create_notification(template=sample_template, created_at=datetime(2017, 1, 15), billable_units=2, + status='delivered') + create_notification(template=sample_template, created_at=datetime(2017, 1, 17, 13, 30, 57), billable_units=4, + status='delivered') + + create_or_update_monthly_billing_sms(service_id=sample_template.service_id, + billing_month=rate_2) + monthly_billing = MonthlyBilling.query.all() + assert len(monthly_billing) == 1 + assert monthly_billing[0].month == 'January' + + january = get_monthly_billing_sms(service_id=sample_template.service_id, billing_month=rate_2) + first_row = {"billing_units": 2, + "rate_multiplier": 1, + "international": False, + "rate": 0.0158, + "total_cost": 3 * 0.0158} + assert_monthly_billing(january, 2017, "January", sample_template.service_id, 2, first_row) + second_row = {"billing_units": 6, + "rate_multiplier": 1, + "international": False, + "rate": 0.0124, + "total_cost": 1 * 0.0124} + assert sorted(january.monthly_totals[1]) == sorted(second_row) + + +def test_update_monthly_billing_overwrites_old_totals(sample_template): + july = datetime(2017, 7, 1) + create_rate(july, 0.123, 'sms') + create_notification(template=sample_template, created_at=datetime(2017, 7, 2), billable_units=1, status='delivered') + + create_or_update_monthly_billing_sms(sample_template.service_id, july) + first_update = get_monthly_billing_sms(sample_template.service_id, july) + expected = {"billing_units": 1, + "rate_multiplier": 1, + "international": False, + "rate": 0.123, + "total_cost": 1 * 0.123} + assert_monthly_billing(first_update, 2017, "July", sample_template.service_id, 1, expected) + + create_notification(template=sample_template, created_at=datetime(2017, 7, 5), billable_units=2, status='delivered') + create_or_update_monthly_billing_sms(sample_template.service_id, july) + second_update = get_monthly_billing_sms(sample_template.service_id, july) + expected_update = {"billing_units": 3, + "rate_multiplier": 1, + "international": False, + "rate": 0.123, + "total_cost": 3 * 0.123} + assert_monthly_billing(second_update, 2017, "July", sample_template.service_id, 1, expected_update) + + +def assert_monthly_billing(monthly_billing, year, month, service_id, expected_len, first_row): + assert monthly_billing.year == year + assert monthly_billing.month == month + assert monthly_billing.service_id == service_id + assert len(monthly_billing.monthly_totals) == expected_len + assert sorted(monthly_billing.monthly_totals[0]) == sorted(first_row) diff --git a/tests/app/dao/test_provider_rates_dao.py b/tests/app/dao/test_provider_rates_dao.py index 417612781..7edb7de43 100644 --- a/tests/app/dao/test_provider_rates_dao.py +++ b/tests/app/dao/test_provider_rates_dao.py @@ -1,7 +1,8 @@ +import uuid from datetime import datetime from decimal import Decimal -from app.dao.provider_rates_dao import create_provider_rates -from app.models import ProviderRates, ProviderDetails +from app.dao.provider_rates_dao import create_provider_rates, create_sms_rate +from app.models import ProviderRates, ProviderDetails, Rate def test_create_provider_rates(notify_db, notify_db_session, mmg_provider): @@ -15,3 +16,11 @@ def test_create_provider_rates(notify_db, notify_db_session, mmg_provider): assert ProviderRates.query.first().rate == rate assert ProviderRates.query.first().valid_from == now assert ProviderRates.query.first().provider_id == provider.id + + +def test_create_sms_rate(): + rate = Rate(id=uuid.uuid4(), valid_from=datetime.now(), rate=0.014, notification_type='sms') + create_sms_rate(rate) + rates = Rate.query.all() + assert len(rates) == 1 + assert rates[0] == rate diff --git a/tests/app/db.py b/tests/app/db.py index ea9e4cbdd..669803d92 100644 --- a/tests/app/db.py +++ b/tests/app/db.py @@ -1,8 +1,8 @@ from datetime import datetime import uuid - from app.dao.jobs_dao import dao_create_job +from app.dao.provider_rates_dao import create_sms_rate from app.dao.service_inbound_api_dao import save_service_inbound_api from app.models import ( Service, @@ -11,6 +11,7 @@ from app.models import ( Notification, ScheduledNotification, ServicePermission, + Rate, Job, InboundSms, Organisation, @@ -239,3 +240,9 @@ def create_organisation(colour='blue', logo='test_x2.png', name='test_org_1'): dao_create_organisation(organisation) return organisation + + +def create_rate(start_date, value, notification_type): + rate = Rate(id=uuid.uuid4(), valid_from=start_date, rate=value, notification_type=notification_type) + create_sms_rate(rate) + return rate From 793248a74f0da249bbe9e23582b00dff953919aa Mon Sep 17 00:00:00 2001 From: Rebecca Law Date: Wed, 19 Jul 2017 15:47:12 +0100 Subject: [PATCH 3/6] Fix data migration merge conflict --- .../{0109_monthly_billing.py => 0110_monthly_billing.py} | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) rename migrations/versions/{0109_monthly_billing.py => 0110_monthly_billing.py} (89%) diff --git a/migrations/versions/0109_monthly_billing.py b/migrations/versions/0110_monthly_billing.py similarity index 89% rename from migrations/versions/0109_monthly_billing.py rename to migrations/versions/0110_monthly_billing.py index 124a509cf..19b1ce4fc 100644 --- a/migrations/versions/0109_monthly_billing.py +++ b/migrations/versions/0110_monthly_billing.py @@ -1,14 +1,14 @@ """empty message -Revision ID: 0109_monthly_billing -Revises: 0108_change_logo_not_nullable +Revision ID: 0110_monthly_billing +Revises: 0109_rem_old_noti_status Create Date: 2017-07-13 14:35:03.183659 """ # revision identifiers, used by Alembic. -revision = '0109_monthly_billing' -down_revision = '0108_change_logo_not_nullable' +revision = '0110_monthly_billing' +down_revision = '0109_rem_old_noti_status' from alembic import op import sqlalchemy as sa From 3e2b8190b9262734f0849e33ff778f69be034e6e Mon Sep 17 00:00:00 2001 From: Rebecca Law Date: Mon, 24 Jul 2017 15:13:18 +0100 Subject: [PATCH 4/6] - Added a scheduled task to create or update billing for the month, yesterday is used to calculate the start and end date for the month. - The new task has not been added to the beat application yet. - Added an updated_at column to the monthly billing table, we may want to only calculate from the last updated date rather than the entire month. --- app/celery/scheduled_tasks.py | 21 ++++++++++- app/dao/monthly_billing_dao.py | 17 +++++++-- app/dao/notification_usage_dao.py | 3 +- app/models.py | 1 + migrations/versions/0110_monthly_billing.py | 1 + tests/app/celery/test_scheduled_tasks.py | 39 ++++++++++++++++++--- tests/app/dao/test_monthly_billing.py | 23 ++++++++++-- 7 files changed, 93 insertions(+), 12 deletions(-) diff --git a/app/celery/scheduled_tasks.py b/app/celery/scheduled_tasks.py index b14f7a5be..ceb813e25 100644 --- a/app/celery/scheduled_tasks.py +++ b/app/celery/scheduled_tasks.py @@ -9,9 +9,17 @@ from sqlalchemy.exc import SQLAlchemyError from app.aws import s3 from app import notify_celery from app import performance_platform_client +from app.dao.date_util import get_month_start_end_date from app.dao.inbound_sms_dao import delete_inbound_sms_created_more_than_a_week_ago from app.dao.invited_user_dao import delete_invitations_created_more_than_two_days_ago -from app.dao.jobs_dao import dao_set_scheduled_jobs_to_pending, dao_get_jobs_older_than_limited_by +from app.dao.jobs_dao import ( + dao_set_scheduled_jobs_to_pending, + dao_get_jobs_older_than_limited_by +) +from app.dao.monthly_billing_dao import ( + get_service_ids_that_need_sms_billing_populated, + create_or_update_monthly_billing_sms +) from app.dao.notifications_dao import ( dao_timeout_notifications, is_delivery_slow_for_provider, @@ -281,3 +289,14 @@ def delete_dvla_response_files_older_than_seven_days(): except SQLAlchemyError as e: current_app.logger.exception("Failed to delete dvla response files") raise + + +@notify_celery.task(name="populate_monthly_billing") +@statsd(namespace="tasks") +def populate_monthly_billing(): + # for every service with billable units this month update billing totals for yesterday + # this will overwrite the existing amount. + yesterday = datetime.utcnow() - timedelta(days=1) + start_date, end_date = get_month_start_end_date(yesterday) + services = get_service_ids_that_need_sms_billing_populated(start_date, end_date=end_date) + [create_or_update_monthly_billing_sms(service_id=s.service_id, billing_month=start_date) for s in services] diff --git a/app/dao/monthly_billing_dao.py b/app/dao/monthly_billing_dao.py index 8e3204584..4dede3f2d 100644 --- a/app/dao/monthly_billing_dao.py +++ b/app/dao/monthly_billing_dao.py @@ -1,12 +1,25 @@ from datetime import datetime from app import db +from app.dao.date_util import get_month_start_end_date from app.dao.notification_usage_dao import get_billing_data_for_month -from app.models import MonthlyBilling, SMS_TYPE +from app.models import MonthlyBilling, SMS_TYPE, NotificationHistory + + +def get_service_ids_that_need_sms_billing_populated(start_date, end_date): + return db.session.query( + NotificationHistory.service_id + ).filter( + NotificationHistory.created_at >= start_date, + NotificationHistory.created_at <= end_date, + NotificationHistory.notification_type == SMS_TYPE, + NotificationHistory.billable_units != 0 + ).distinct().all() def create_or_update_monthly_billing_sms(service_id, billing_month): - monthly = get_billing_data_for_month(service_id=service_id, billing_month=billing_month) + start_date, end_date = get_month_start_end_date(billing_month) + monthly = get_billing_data_for_month(service_id=service_id, start_date=start_date, end_date=end_date) # update monthly monthly_totals = _monthly_billing_data_to_json(monthly) row = MonthlyBilling.query.filter_by(year=billing_month.year, diff --git a/app/dao/notification_usage_dao.py b/app/dao/notification_usage_dao.py index 289a5a03d..13b8944c6 100644 --- a/app/dao/notification_usage_dao.py +++ b/app/dao/notification_usage_dao.py @@ -36,8 +36,7 @@ def get_yearly_billing_data(service_id, year): @statsd(namespace="dao") -def get_billing_data_for_month(service_id, billing_month): - start_date, end_date = get_month_start_end_date(billing_month) +def get_billing_data_for_month(service_id, start_date, end_date): rates = get_rates_for_year(start_date, end_date, SMS_TYPE) result = [] # so the start end date in the query are the valid from the rate, not the month - this is going to take some thought diff --git a/app/models.py b/app/models.py index e035acccb..b58eaf348 100644 --- a/app/models.py +++ b/app/models.py @@ -1257,6 +1257,7 @@ class MonthlyBilling(db.Model): year = db.Column(db.Float(asdecimal=False), nullable=False) notification_type = db.Column(notification_types, nullable=False) monthly_totals = db.Column(JSON, nullable=False) + updated_at = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow) __table_args__ = ( UniqueConstraint('service_id', 'month', 'year', 'notification_type', name='uix_monthly_billing'), diff --git a/migrations/versions/0110_monthly_billing.py b/migrations/versions/0110_monthly_billing.py index 19b1ce4fc..19fa1dbdd 100644 --- a/migrations/versions/0110_monthly_billing.py +++ b/migrations/versions/0110_monthly_billing.py @@ -26,6 +26,7 @@ def upgrade(): postgresql.ENUM('email', 'sms', 'letter', name='notification_type', create_type=False), nullable=False), sa.Column('monthly_totals', postgresql.JSON(), nullable=False), + sa.Column('updated_at', sa.DateTime, nullable=False), sa.ForeignKeyConstraint(['service_id'], ['services.id'], ), sa.PrimaryKeyConstraint('id') ) diff --git a/tests/app/celery/test_scheduled_tasks.py b/tests/app/celery/test_scheduled_tasks.py index b4ad7e255..0f54f9dde 100644 --- a/tests/app/celery/test_scheduled_tasks.py +++ b/tests/app/celery/test_scheduled_tasks.py @@ -25,8 +25,8 @@ from app.celery.scheduled_tasks import ( send_scheduled_notifications, switch_current_sms_provider_on_slow_delivery, timeout_job_statistics, - timeout_notifications -) + timeout_notifications, + populate_monthly_billing) from app.clients.performance_platform.performance_platform_client import PerformancePlatformClient from app.dao.jobs_dao import dao_get_job_by_id from app.dao.notifications_dao import dao_get_scheduled_notifications @@ -36,10 +36,10 @@ from app.dao.provider_details_dao import ( ) from app.models import ( Service, Template, - SMS_TYPE, LETTER_TYPE -) + SMS_TYPE, LETTER_TYPE, + MonthlyBilling) from app.utils import get_london_midnight_in_utc -from tests.app.db import create_notification, create_service, create_template, create_job +from tests.app.db import create_notification, create_service, create_template, create_job, create_rate from tests.app.conftest import ( sample_job as create_sample_job, sample_notification_history as create_notification_history, @@ -98,6 +98,8 @@ def test_should_have_decorated_tasks_functions(): 'remove_transformed_dvla_files' assert delete_dvla_response_files_older_than_seven_days.__wrapped__.__name__ == \ 'delete_dvla_response_files_older_than_seven_days' + assert populate_monthly_billing.__wrapped__.__name__ == \ + 'populate_monthly_billing' @pytest.fixture(scope='function') @@ -607,3 +609,30 @@ def test_delete_dvla_response_files_older_than_seven_days_does_not_remove_files( delete_dvla_response_files_older_than_seven_days() remove_s3_mock.assert_not_called() + + +@freeze_time("2017-07-12 02:00:00") +def test_populate_monthly_billing(sample_template): + yesterday = datetime(2017, 7, 11, 13, 30) + create_rate(datetime(2016, 1, 1), 0.0123, 'sms') + create_notification(template=sample_template, status='delivered', created_at=yesterday) + create_notification(template=sample_template, status='delivered', created_at=yesterday - timedelta(days=1)) + create_notification(template=sample_template, status='delivered', created_at=yesterday + timedelta(days=1)) + # not included in billing + create_notification(template=sample_template, status='delivered', created_at=yesterday - timedelta(days=30)) + + assert len(MonthlyBilling.query.all()) == 0 + populate_monthly_billing() + + monthly_billing = MonthlyBilling.query.all() + assert len(monthly_billing) == 1 + assert monthly_billing[0].service_id == sample_template.service_id + assert monthly_billing[0].year == 2017 + assert monthly_billing[0].month == 'July' + assert monthly_billing[0].notification_type == 'sms' + assert len(monthly_billing[0].monthly_totals) == 1 + assert sorted(monthly_billing[0].monthly_totals[0]) == sorted({'international': False, + 'rate_multiplier': 1, + 'billing_units': 3, + 'rate': 0.0123, + 'total_cost': 0.0369}) diff --git a/tests/app/dao/test_monthly_billing.py b/tests/app/dao/test_monthly_billing.py index 6a0399d67..2ed13aaa2 100644 --- a/tests/app/dao/test_monthly_billing.py +++ b/tests/app/dao/test_monthly_billing.py @@ -1,8 +1,12 @@ from datetime import datetime -from app.dao.monthly_billing_dao import create_or_update_monthly_billing_sms, get_monthly_billing_sms +from app.dao.monthly_billing_dao import ( + create_or_update_monthly_billing_sms, + get_monthly_billing_sms, + get_service_ids_that_need_sms_billing_populated +) from app.models import MonthlyBilling -from tests.app.db import create_notification, create_rate +from tests.app.db import create_notification, create_rate, create_service, create_template def test_add_monthly_billing(sample_template): @@ -105,3 +109,18 @@ def assert_monthly_billing(monthly_billing, year, month, service_id, expected_le assert monthly_billing.service_id == service_id assert len(monthly_billing.monthly_totals) == expected_len assert sorted(monthly_billing.monthly_totals[0]) == sorted(first_row) + + +def test_get_service_id(): + service_1 = create_service(service_name="Service One") + template_1 = create_template(service=service_1) + service_2 = create_service(service_name="Service Two") + template_2 = create_template(service=service_2) + create_notification(template=template_1, created_at=datetime(2017, 6, 30, 13, 30), status='delivered') + create_notification(template=template_1, created_at=datetime(2017, 7, 1, 14, 30), status='delivered') + create_notification(template=template_2, created_at=datetime(2017, 7, 15, 13, 30)) + create_notification(template=template_2, created_at=datetime(2017, 7, 31, 13, 30)) + services = get_service_ids_that_need_sms_billing_populated(start_date=datetime(2017, 7, 1), + end_date=datetime(2017, 7, 16)) + expected_services = [service_1.id, service_2.id] + assert sorted([x.service_id for x in services]) == sorted(expected_services) From eaf5cbb86876c417e6dec4d6b76e24f94cbe71bd Mon Sep 17 00:00:00 2001 From: Rebecca Law Date: Tue, 25 Jul 2017 11:43:41 +0100 Subject: [PATCH 5/6] Add labels to query so that the named tuples can be referenced later. Remove unnecessary function --- app/dao/monthly_billing_dao.py | 10 +++++----- app/dao/notification_usage_dao.py | 8 ++++---- app/dao/provider_rates_dao.py | 5 ----- tests/app/dao/test_monthly_billing.py | 2 +- tests/app/dao/test_provider_rates_dao.py | 12 ++---------- tests/app/db.py | 5 +++-- 6 files changed, 15 insertions(+), 27 deletions(-) diff --git a/app/dao/monthly_billing_dao.py b/app/dao/monthly_billing_dao.py index 4dede3f2d..6cc499001 100644 --- a/app/dao/monthly_billing_dao.py +++ b/app/dao/monthly_billing_dao.py @@ -50,8 +50,8 @@ def _monthly_billing_data_to_json(monthly): # (month, billing_units, rate_multiplier, international, notification_type, rate) # total cost must take into account the free allowance. # might be a good idea to capture free allowance in this table - return [{"billing_units": x[1], - "rate_multiplier": x[2], - "international": x[3], - "rate": x[5], - "total_cost": (x[1] * x[2]) * x[5]} for x in monthly] + return [{"billing_units": x.billing_units, + "rate_multiplier": x.rate_multiplier, + "international": x.international, + "rate": x.rate, + "total_cost": (x.billing_units * x.rate_multiplier) * x.rate} for x in monthly] diff --git a/app/dao/notification_usage_dao.py b/app/dao/notification_usage_dao.py index 13b8944c6..43fb6a6cd 100644 --- a/app/dao/notification_usage_dao.py +++ b/app/dao/notification_usage_dao.py @@ -142,12 +142,12 @@ def is_between(date, start_date, end_date): def sms_billing_data_per_month_query(rate, service_id, start_date, end_date): month = get_london_month_from_utc_column(NotificationHistory.created_at) result = db.session.query( - month, - func.sum(NotificationHistory.billable_units), - rate_multiplier(), + month.label('month'), + func.sum(NotificationHistory.billable_units).label('billing_units'), + rate_multiplier().label('rate_multiplier'), NotificationHistory.international, NotificationHistory.notification_type, - cast(rate, Float()) + cast(rate, Float()).label('rate') ).filter( *billing_data_filter(SMS_TYPE, start_date, end_date, service_id) ).group_by( diff --git a/app/dao/provider_rates_dao.py b/app/dao/provider_rates_dao.py index 443543f86..145bd431e 100644 --- a/app/dao/provider_rates_dao.py +++ b/app/dao/provider_rates_dao.py @@ -9,8 +9,3 @@ def create_provider_rates(provider_identifier, valid_from, rate): provider_rates = ProviderRates(provider_id=provider.id, valid_from=valid_from, rate=rate) db.session.add(provider_rates) - - -@transactional -def create_sms_rate(rate): - db.session.add(rate) diff --git a/tests/app/dao/test_monthly_billing.py b/tests/app/dao/test_monthly_billing.py index 2ed13aaa2..9027ff870 100644 --- a/tests/app/dao/test_monthly_billing.py +++ b/tests/app/dao/test_monthly_billing.py @@ -111,7 +111,7 @@ def assert_monthly_billing(monthly_billing, year, month, service_id, expected_le assert sorted(monthly_billing.monthly_totals[0]) == sorted(first_row) -def test_get_service_id(): +def test_get_service_id(notify_db_session): service_1 = create_service(service_name="Service One") template_1 = create_template(service=service_1) service_2 = create_service(service_name="Service Two") diff --git a/tests/app/dao/test_provider_rates_dao.py b/tests/app/dao/test_provider_rates_dao.py index 7edb7de43..c78290a90 100644 --- a/tests/app/dao/test_provider_rates_dao.py +++ b/tests/app/dao/test_provider_rates_dao.py @@ -1,8 +1,8 @@ import uuid from datetime import datetime from decimal import Decimal -from app.dao.provider_rates_dao import create_provider_rates, create_sms_rate -from app.models import ProviderRates, ProviderDetails, Rate +from app.dao.provider_rates_dao import create_provider_rates +from app.models import ProviderRates, ProviderDetails def test_create_provider_rates(notify_db, notify_db_session, mmg_provider): @@ -16,11 +16,3 @@ def test_create_provider_rates(notify_db, notify_db_session, mmg_provider): assert ProviderRates.query.first().rate == rate assert ProviderRates.query.first().valid_from == now assert ProviderRates.query.first().provider_id == provider.id - - -def test_create_sms_rate(): - rate = Rate(id=uuid.uuid4(), valid_from=datetime.now(), rate=0.014, notification_type='sms') - create_sms_rate(rate) - rates = Rate.query.all() - assert len(rates) == 1 - assert rates[0] == rate diff --git a/tests/app/db.py b/tests/app/db.py index 669803d92..ace7b7c1c 100644 --- a/tests/app/db.py +++ b/tests/app/db.py @@ -1,8 +1,8 @@ from datetime import datetime import uuid +from app import db from app.dao.jobs_dao import dao_create_job -from app.dao.provider_rates_dao import create_sms_rate from app.dao.service_inbound_api_dao import save_service_inbound_api from app.models import ( Service, @@ -244,5 +244,6 @@ def create_organisation(colour='blue', logo='test_x2.png', name='test_org_1'): def create_rate(start_date, value, notification_type): rate = Rate(id=uuid.uuid4(), valid_from=start_date, rate=value, notification_type=notification_type) - create_sms_rate(rate) + db.session.add(rate) + db.session.commit() return rate From d2a1da9ea6cf134f51ef73280a0e09af9033e8f2 Mon Sep 17 00:00:00 2001 From: Rebecca Law Date: Tue, 25 Jul 2017 11:44:39 +0100 Subject: [PATCH 6/6] Removed comment --- app/dao/monthly_billing_dao.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/dao/monthly_billing_dao.py b/app/dao/monthly_billing_dao.py index 6cc499001..cd51c19d3 100644 --- a/app/dao/monthly_billing_dao.py +++ b/app/dao/monthly_billing_dao.py @@ -46,8 +46,6 @@ def get_monthly_billing_sms(service_id, billing_month): def _monthly_billing_data_to_json(monthly): - # ('April', 6, 1, False, 'sms', 0.014) - # (month, billing_units, rate_multiplier, international, notification_type, rate) # total cost must take into account the free allowance. # might be a good idea to capture free allowance in this table return [{"billing_units": x.billing_units,