diff --git a/app/celery/scheduled_tasks.py b/app/celery/scheduled_tasks.py index e86a7c113..08433e017 100644 --- a/app/celery/scheduled_tasks.py +++ b/app/celery/scheduled_tasks.py @@ -9,9 +9,17 @@ from sqlalchemy.exc import SQLAlchemyError from app.aws import s3 from app import notify_celery from app import performance_platform_client +from app.dao.date_util import get_month_start_end_date from app.dao.inbound_sms_dao import delete_inbound_sms_created_more_than_a_week_ago from app.dao.invited_user_dao import delete_invitations_created_more_than_two_days_ago -from app.dao.jobs_dao import dao_set_scheduled_jobs_to_pending, dao_get_jobs_older_than_limited_by +from app.dao.jobs_dao import ( + dao_set_scheduled_jobs_to_pending, + dao_get_jobs_older_than_limited_by +) +from app.dao.monthly_billing_dao import ( + get_service_ids_that_need_sms_billing_populated, + create_or_update_monthly_billing_sms +) from app.dao.notifications_dao import ( dao_timeout_notifications, is_delivery_slow_for_provider, @@ -281,3 +289,14 @@ def delete_dvla_response_files_older_than_seven_days(): except SQLAlchemyError as e: current_app.logger.exception("Failed to delete dvla response files") raise + + +@notify_celery.task(name="populate_monthly_billing") +@statsd(namespace="tasks") +def populate_monthly_billing(): + # for every service with billable units this month update billing totals for yesterday + # this will overwrite the existing amount. + yesterday = datetime.utcnow() - timedelta(days=1) + start_date, end_date = get_month_start_end_date(yesterday) + services = get_service_ids_that_need_sms_billing_populated(start_date, end_date=end_date) + [create_or_update_monthly_billing_sms(service_id=s.service_id, billing_month=start_date) for s in services] diff --git a/app/commands.py b/app/commands.py index 6e4bc53d1..c7ec0567b 100644 --- a/app/commands.py +++ b/app/commands.py @@ -5,7 +5,8 @@ from flask.ext.script import Command, Manager, Option from app import db -from app.models import (PROVIDERS, Service, User, NotificationHistory) +from app.dao.monthly_billing_dao import create_or_update_monthly_billing_sms, get_monthly_billing_sms +from app.models import (PROVIDERS, User) from app.dao.services_dao import ( delete_service_and_all_associated_db_objects, dao_fetch_all_services_by_user @@ -146,3 +147,18 @@ class CustomDbScript(Command): print('Committed {} updates at {}'.format(len(result), datetime.utcnow())) db.session.commit() result = db.session.execute(subq_hist).fetchall() + + +class PopulateMonthlyBilling(Command): + option_list = ( + Option('-s', '-service-id', dest='service_id', + help="Service id to populate monthly billing for"), + Option('-m', '-month', dest="month", help="Use for integer value for month, e.g. 7 for July"), + Option('-y', '-year', dest="year", help="Use for integer value for year, e.g. 2017") + ) + + def run(self, service_id, month, year): + create_or_update_monthly_billing_sms(service_id, datetime(int(year), int(month), 1)) + results = get_monthly_billing_sms(service_id, datetime(int(year), int(month), 1)) + print("Finished populating data for {} for service id {}".format(month, service_id)) + print(results.monthly_totals) diff --git a/app/dao/date_util.py b/app/dao/date_util.py index 2471865bd..41ad9dc72 100644 --- a/app/dao/date_util.py +++ b/app/dao/date_util.py @@ -16,3 +16,16 @@ def get_april_fools(year): """ return pytz.timezone('Europe/London').localize(datetime(year, 4, 1, 0, 0, 0)).astimezone(pytz.UTC).replace( tzinfo=None) + + +def get_month_start_end_date(month_year): + """ + This function return the start and date of the month_year as UTC, + :param month_year: the datetime to calculate the start and end date for that month + :return: start_date, end_date, month + """ + import calendar + _, num_days = calendar.monthrange(month_year.year, month_year.month) + first_day = datetime(month_year.year, month_year.month, 1, 0, 0, 0) + last_day = datetime(month_year.year, month_year.month, num_days, 23, 59, 59, 99999) + return first_day, last_day diff --git a/app/dao/monthly_billing_dao.py b/app/dao/monthly_billing_dao.py new file mode 100644 index 000000000..cd51c19d3 --- /dev/null +++ b/app/dao/monthly_billing_dao.py @@ -0,0 +1,55 @@ +from datetime import datetime + +from app import db +from app.dao.date_util import get_month_start_end_date +from app.dao.notification_usage_dao import get_billing_data_for_month +from app.models import MonthlyBilling, SMS_TYPE, NotificationHistory + + +def get_service_ids_that_need_sms_billing_populated(start_date, end_date): + return db.session.query( + NotificationHistory.service_id + ).filter( + NotificationHistory.created_at >= start_date, + NotificationHistory.created_at <= end_date, + NotificationHistory.notification_type == SMS_TYPE, + NotificationHistory.billable_units != 0 + ).distinct().all() + + +def create_or_update_monthly_billing_sms(service_id, billing_month): + start_date, end_date = get_month_start_end_date(billing_month) + monthly = get_billing_data_for_month(service_id=service_id, start_date=start_date, end_date=end_date) + # update monthly + monthly_totals = _monthly_billing_data_to_json(monthly) + row = MonthlyBilling.query.filter_by(year=billing_month.year, + month=datetime.strftime(billing_month, "%B"), + notification_type='sms').first() + if row: + row.monthly_totals = monthly_totals + else: + row = MonthlyBilling(service_id=service_id, + notification_type=SMS_TYPE, + year=billing_month.year, + month=datetime.strftime(billing_month, "%B"), + monthly_totals=monthly_totals) + db.session.add(row) + db.session.commit() + + +def get_monthly_billing_sms(service_id, billing_month): + monthly = MonthlyBilling.query.filter_by(service_id=service_id, + year=billing_month.year, + month=datetime.strftime(billing_month, "%B"), + notification_type=SMS_TYPE).first() + return monthly + + +def _monthly_billing_data_to_json(monthly): + # total cost must take into account the free allowance. + # might be a good idea to capture free allowance in this table + return [{"billing_units": x.billing_units, + "rate_multiplier": x.rate_multiplier, + "international": x.international, + "rate": x.rate, + "total_cost": (x.billing_units * x.rate_multiplier) * x.rate} for x in monthly] diff --git a/app/dao/notification_usage_dao.py b/app/dao/notification_usage_dao.py index 995462509..43fb6a6cd 100644 --- a/app/dao/notification_usage_dao.py +++ b/app/dao/notification_usage_dao.py @@ -6,7 +6,7 @@ from sqlalchemy import func, case, cast from sqlalchemy import literal_column from app import db -from app.dao.date_util import get_financial_year +from app.dao.date_util import get_financial_year, get_month_start_end_date from app.models import (NotificationHistory, Rate, NOTIFICATION_STATUS_TYPES_BILLABLE, @@ -35,6 +35,20 @@ def get_yearly_billing_data(service_id, year): return sum(result, []) +@statsd(namespace="dao") +def get_billing_data_for_month(service_id, start_date, end_date): + rates = get_rates_for_year(start_date, end_date, SMS_TYPE) + result = [] + # so the start end date in the query are the valid from the rate, not the month - this is going to take some thought + for r, n in zip(rates, rates[1:]): + result.extend(sms_billing_data_per_month_query(r.rate, service_id, max(r.valid_from, start_date), + min(n.valid_from, end_date))) + result.extend( + sms_billing_data_per_month_query(rates[-1].rate, service_id, max(rates[-1].valid_from, start_date), end_date)) + + return result + + @statsd(namespace="dao") def get_monthly_billing_data(service_id, year): start_date, end_date = get_financial_year(year) @@ -128,12 +142,12 @@ def is_between(date, start_date, end_date): def sms_billing_data_per_month_query(rate, service_id, start_date, end_date): month = get_london_month_from_utc_column(NotificationHistory.created_at) result = db.session.query( - month, - func.sum(NotificationHistory.billable_units), - rate_multiplier(), + month.label('month'), + func.sum(NotificationHistory.billable_units).label('billing_units'), + rate_multiplier().label('rate_multiplier'), NotificationHistory.international, NotificationHistory.notification_type, - cast(rate, Float()) + cast(rate, Float()).label('rate') ).filter( *billing_data_filter(SMS_TYPE, start_date, end_date, service_id) ).group_by( diff --git a/app/delivery/send_to_providers.py b/app/delivery/send_to_providers.py index df4c664f9..5e33e2113 100644 --- a/app/delivery/send_to_providers.py +++ b/app/delivery/send_to_providers.py @@ -18,7 +18,7 @@ from app.dao.templates_dao import dao_get_template_by_id from app.models import SMS_TYPE, KEY_TYPE_TEST, BRANDING_ORG, EMAIL_TYPE, NOTIFICATION_TECHNICAL_FAILURE, \ NOTIFICATION_SENT, NOTIFICATION_SENDING -from app.celery.statistics_tasks import record_initial_job_statistics, create_initial_notification_statistic_tasks +from app.celery.statistics_tasks import create_initial_notification_statistic_tasks def send_sms_to_provider(notification): diff --git a/app/models.py b/app/models.py index 0e3079c80..b58eaf348 100644 --- a/app/models.py +++ b/app/models.py @@ -4,7 +4,6 @@ import datetime from flask import url_for, current_app from sqlalchemy.ext.associationproxy import association_proxy -from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.dialects.postgresql import ( UUID, JSON @@ -1246,3 +1245,29 @@ class LetterRateDetail(db.Model): letter_rate = db.relationship('LetterRate', backref='letter_rates') page_total = db.Column(db.Integer, nullable=False) rate = db.Column(db.Numeric(), nullable=False) + + +class MonthlyBilling(db.Model): + __tablename__ = 'monthly_billing' + + id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'), index=True, nullable=False) + service = db.relationship('Service', backref='monthly_billing') + month = db.Column(db.String, nullable=False) + year = db.Column(db.Float(asdecimal=False), nullable=False) + notification_type = db.Column(notification_types, nullable=False) + monthly_totals = db.Column(JSON, nullable=False) + updated_at = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow) + + __table_args__ = ( + UniqueConstraint('service_id', 'month', 'year', 'notification_type', name='uix_monthly_billing'), + ) + + def serialized(self): + return { + "month": self.month, + "year": self.year, + "service_id": str(self.service_id), + "notification_type": self.notification_type, + "monthly_totals": self.monthly_totals + } diff --git a/app/service/rest.py b/app/service/rest.py index ae57f5390..deb37f525 100644 --- a/app/service/rest.py +++ b/app/service/rest.py @@ -68,7 +68,6 @@ from app.schemas import ( user_schema, permission_schema, notification_with_template_schema, - notification_with_personalisation_schema, notifications_filter_schema, detailed_service_schema ) diff --git a/application.py b/application.py index b2a59a6fd..5ff7dd1e4 100644 --- a/application.py +++ b/application.py @@ -16,6 +16,7 @@ manager.add_command('db', MigrateCommand) manager.add_command('create_provider_rate', commands.CreateProviderRateCommand) manager.add_command('purge_functional_test_data', commands.PurgeFunctionalTestDataCommand) manager.add_command('custom_db_script', commands.CustomDbScript) +manager.add_command('populate_monthly_billing', commands.PopulateMonthlyBilling) @manager.command diff --git a/migrations/versions/0110_monthly_billing.py b/migrations/versions/0110_monthly_billing.py new file mode 100644 index 000000000..19fa1dbdd --- /dev/null +++ b/migrations/versions/0110_monthly_billing.py @@ -0,0 +1,38 @@ +"""empty message + +Revision ID: 0110_monthly_billing +Revises: 0109_rem_old_noti_status +Create Date: 2017-07-13 14:35:03.183659 + +""" + +# revision identifiers, used by Alembic. +revision = '0110_monthly_billing' +down_revision = '0109_rem_old_noti_status' + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + + +def upgrade(): + + op.create_table('monthly_billing', + sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('service_id', postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('month', sa.String(), nullable=False), + sa.Column('year', sa.Float(), nullable=False), + sa.Column('notification_type', + postgresql.ENUM('email', 'sms', 'letter', name='notification_type', create_type=False), + nullable=False), + sa.Column('monthly_totals', postgresql.JSON(), nullable=False), + sa.Column('updated_at', sa.DateTime, nullable=False), + sa.ForeignKeyConstraint(['service_id'], ['services.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_monthly_billing_service_id'), 'monthly_billing', ['service_id'], unique=False) + op.create_index(op.f('uix_monthly_billing'), 'monthly_billing', ['service_id', 'month', 'year', 'notification_type'], unique=True) + + +def downgrade(): + op.drop_table('monthly_billing') diff --git a/tests/app/celery/test_scheduled_tasks.py b/tests/app/celery/test_scheduled_tasks.py index 0282f8074..6ea42844d 100644 --- a/tests/app/celery/test_scheduled_tasks.py +++ b/tests/app/celery/test_scheduled_tasks.py @@ -25,8 +25,8 @@ from app.celery.scheduled_tasks import ( send_scheduled_notifications, switch_current_sms_provider_on_slow_delivery, timeout_job_statistics, - timeout_notifications -) + timeout_notifications, + populate_monthly_billing) from app.clients.performance_platform.performance_platform_client import PerformancePlatformClient from app.dao.jobs_dao import dao_get_job_by_id from app.dao.notifications_dao import dao_get_scheduled_notifications @@ -36,10 +36,10 @@ from app.dao.provider_details_dao import ( ) from app.models import ( Service, Template, - SMS_TYPE, LETTER_TYPE -) + SMS_TYPE, LETTER_TYPE, + MonthlyBilling) from app.utils import get_london_midnight_in_utc -from tests.app.db import create_notification, create_service, create_template, create_job +from tests.app.db import create_notification, create_service, create_template, create_job, create_rate from tests.app.conftest import ( sample_job as create_sample_job, sample_notification_history as create_notification_history, @@ -98,6 +98,8 @@ def test_should_have_decorated_tasks_functions(): 'remove_transformed_dvla_files' assert delete_dvla_response_files_older_than_seven_days.__wrapped__.__name__ == \ 'delete_dvla_response_files_older_than_seven_days' + assert populate_monthly_billing.__wrapped__.__name__ == \ + 'populate_monthly_billing' @pytest.fixture(scope='function') @@ -607,3 +609,30 @@ def test_delete_dvla_response_files_older_than_seven_days_does_not_remove_files( delete_dvla_response_files_older_than_seven_days() remove_s3_mock.assert_not_called() + + +@freeze_time("2017-07-12 02:00:00") +def test_populate_monthly_billing(sample_template): + yesterday = datetime(2017, 7, 11, 13, 30) + create_rate(datetime(2016, 1, 1), 0.0123, 'sms') + create_notification(template=sample_template, status='delivered', created_at=yesterday) + create_notification(template=sample_template, status='delivered', created_at=yesterday - timedelta(days=1)) + create_notification(template=sample_template, status='delivered', created_at=yesterday + timedelta(days=1)) + # not included in billing + create_notification(template=sample_template, status='delivered', created_at=yesterday - timedelta(days=30)) + + assert len(MonthlyBilling.query.all()) == 0 + populate_monthly_billing() + + monthly_billing = MonthlyBilling.query.all() + assert len(monthly_billing) == 1 + assert monthly_billing[0].service_id == sample_template.service_id + assert monthly_billing[0].year == 2017 + assert monthly_billing[0].month == 'July' + assert monthly_billing[0].notification_type == 'sms' + assert len(monthly_billing[0].monthly_totals) == 1 + assert sorted(monthly_billing[0].monthly_totals[0]) == sorted({'international': False, + 'rate_multiplier': 1, + 'billing_units': 3, + 'rate': 0.0123, + 'total_cost': 0.0369}) diff --git a/tests/app/dao/test_date_utils.py b/tests/app/dao/test_date_utils.py index 760923c5a..4ede18853 100644 --- a/tests/app/dao/test_date_utils.py +++ b/tests/app/dao/test_date_utils.py @@ -1,4 +1,8 @@ -from app.dao.date_util import get_financial_year, get_april_fools +from datetime import datetime + +import pytest + +from app.dao.date_util import get_financial_year, get_april_fools, get_month_start_end_date def test_get_financial_year(): @@ -11,3 +15,16 @@ def test_get_april_fools(): april_fools = get_april_fools(2016) assert str(april_fools) == '2016-03-31 23:00:00' assert april_fools.tzinfo is None + + +@pytest.mark.parametrize("month, year, expected_end", + [(7, 2017, 31), + (2, 2016, 29), + (2, 2017, 28), + (9, 2018, 30), + (12, 2019, 31)]) +def test_get_month_start_end_date(month, year, expected_end): + month_year = datetime(year, month, 10, 13, 30, 00) + result = get_month_start_end_date(month_year) + assert result[0] == datetime(year, month, 1, 0, 0, 0, 0) + assert result[1] == datetime(year, month, expected_end, 23, 59, 59, 99999) diff --git a/tests/app/dao/test_monthly_billing.py b/tests/app/dao/test_monthly_billing.py new file mode 100644 index 000000000..9027ff870 --- /dev/null +++ b/tests/app/dao/test_monthly_billing.py @@ -0,0 +1,126 @@ +from datetime import datetime + +from app.dao.monthly_billing_dao import ( + create_or_update_monthly_billing_sms, + get_monthly_billing_sms, + get_service_ids_that_need_sms_billing_populated +) +from app.models import MonthlyBilling +from tests.app.db import create_notification, create_rate, create_service, create_template + + +def test_add_monthly_billing(sample_template): + jan = datetime(2017, 1, 1) + feb = datetime(2017, 2, 15) + create_rate(start_date=jan, value=0.0158, notification_type='sms') + create_notification(template=sample_template, created_at=jan, billable_units=1, status='delivered') + create_notification(template=sample_template, created_at=feb, billable_units=2, status='delivered') + + create_or_update_monthly_billing_sms(service_id=sample_template.service_id, + billing_month=jan) + create_or_update_monthly_billing_sms(service_id=sample_template.service_id, + billing_month=feb) + monthly_billing = MonthlyBilling.query.all() + assert len(monthly_billing) == 2 + assert monthly_billing[0].month == 'January' + assert monthly_billing[1].month == 'February' + + january = get_monthly_billing_sms(service_id=sample_template.service_id, billing_month=jan) + expected_jan = {"billing_units": 1, + "rate_multiplier": 1, + "international": False, + "rate": 0.0158, + "total_cost": 1 * 0.0158} + assert_monthly_billing(january, 2017, "January", sample_template.service_id, 1, expected_jan) + + february = get_monthly_billing_sms(service_id=sample_template.service_id, billing_month=feb) + expected_feb = {"billing_units": 2, + "rate_multiplier": 1, + "international": False, + "rate": 0.0158, + "total_cost": 2 * 0.0158} + assert_monthly_billing(february, 2017, "February", sample_template.service_id, 1, expected_feb) + + +def test_add_monthly_billing_multiple_rates_in_a_month(sample_template): + rate_1 = datetime(2016, 12, 1) + rate_2 = datetime(2017, 1, 15) + create_rate(start_date=rate_1, value=0.0158, notification_type='sms') + create_rate(start_date=rate_2, value=0.0124, notification_type='sms') + + create_notification(template=sample_template, created_at=datetime(2017, 1, 1), billable_units=1, status='delivered') + create_notification(template=sample_template, created_at=datetime(2017, 1, 14, 23, 59), billable_units=1, + status='delivered') + + create_notification(template=sample_template, created_at=datetime(2017, 1, 15), billable_units=2, + status='delivered') + create_notification(template=sample_template, created_at=datetime(2017, 1, 17, 13, 30, 57), billable_units=4, + status='delivered') + + create_or_update_monthly_billing_sms(service_id=sample_template.service_id, + billing_month=rate_2) + monthly_billing = MonthlyBilling.query.all() + assert len(monthly_billing) == 1 + assert monthly_billing[0].month == 'January' + + january = get_monthly_billing_sms(service_id=sample_template.service_id, billing_month=rate_2) + first_row = {"billing_units": 2, + "rate_multiplier": 1, + "international": False, + "rate": 0.0158, + "total_cost": 3 * 0.0158} + assert_monthly_billing(january, 2017, "January", sample_template.service_id, 2, first_row) + second_row = {"billing_units": 6, + "rate_multiplier": 1, + "international": False, + "rate": 0.0124, + "total_cost": 1 * 0.0124} + assert sorted(january.monthly_totals[1]) == sorted(second_row) + + +def test_update_monthly_billing_overwrites_old_totals(sample_template): + july = datetime(2017, 7, 1) + create_rate(july, 0.123, 'sms') + create_notification(template=sample_template, created_at=datetime(2017, 7, 2), billable_units=1, status='delivered') + + create_or_update_monthly_billing_sms(sample_template.service_id, july) + first_update = get_monthly_billing_sms(sample_template.service_id, july) + expected = {"billing_units": 1, + "rate_multiplier": 1, + "international": False, + "rate": 0.123, + "total_cost": 1 * 0.123} + assert_monthly_billing(first_update, 2017, "July", sample_template.service_id, 1, expected) + + create_notification(template=sample_template, created_at=datetime(2017, 7, 5), billable_units=2, status='delivered') + create_or_update_monthly_billing_sms(sample_template.service_id, july) + second_update = get_monthly_billing_sms(sample_template.service_id, july) + expected_update = {"billing_units": 3, + "rate_multiplier": 1, + "international": False, + "rate": 0.123, + "total_cost": 3 * 0.123} + assert_monthly_billing(second_update, 2017, "July", sample_template.service_id, 1, expected_update) + + +def assert_monthly_billing(monthly_billing, year, month, service_id, expected_len, first_row): + assert monthly_billing.year == year + assert monthly_billing.month == month + assert monthly_billing.service_id == service_id + assert len(monthly_billing.monthly_totals) == expected_len + assert sorted(monthly_billing.monthly_totals[0]) == sorted(first_row) + + +def test_get_service_id(notify_db_session): + service_1 = create_service(service_name="Service One") + template_1 = create_template(service=service_1) + service_2 = create_service(service_name="Service Two") + template_2 = create_template(service=service_2) + create_notification(template=template_1, created_at=datetime(2017, 6, 30, 13, 30), status='delivered') + create_notification(template=template_1, created_at=datetime(2017, 7, 1, 14, 30), status='delivered') + create_notification(template=template_2, created_at=datetime(2017, 7, 15, 13, 30)) + create_notification(template=template_2, created_at=datetime(2017, 7, 31, 13, 30)) + services = get_service_ids_that_need_sms_billing_populated(start_date=datetime(2017, 7, 1), + end_date=datetime(2017, 7, 16)) + expected_services = [service_1.id, service_2.id] + assert sorted([x.service_id for x in services]) == sorted(expected_services) diff --git a/tests/app/dao/test_provider_rates_dao.py b/tests/app/dao/test_provider_rates_dao.py index 417612781..c78290a90 100644 --- a/tests/app/dao/test_provider_rates_dao.py +++ b/tests/app/dao/test_provider_rates_dao.py @@ -1,3 +1,4 @@ +import uuid from datetime import datetime from decimal import Decimal from app.dao.provider_rates_dao import create_provider_rates diff --git a/tests/app/db.py b/tests/app/db.py index ea9e4cbdd..ace7b7c1c 100644 --- a/tests/app/db.py +++ b/tests/app/db.py @@ -1,7 +1,7 @@ from datetime import datetime import uuid - +from app import db from app.dao.jobs_dao import dao_create_job from app.dao.service_inbound_api_dao import save_service_inbound_api from app.models import ( @@ -11,6 +11,7 @@ from app.models import ( Notification, ScheduledNotification, ServicePermission, + Rate, Job, InboundSms, Organisation, @@ -239,3 +240,10 @@ def create_organisation(colour='blue', logo='test_x2.png', name='test_org_1'): dao_create_organisation(organisation) return organisation + + +def create_rate(start_date, value, notification_type): + rate = Rate(id=uuid.uuid4(), valid_from=start_date, rate=value, notification_type=notification_type) + db.session.add(rate) + db.session.commit() + return rate