Merge pull request #1091 from alphagov/month-billing-table

Month billing table

app/celery/scheduled_tasks.py

@@ -9,9 +9,17 @@ from sqlalchemy.exc import SQLAlchemyError
 from app.aws import s3
 from app import notify_celery
 from app import performance_platform_client
+from app.dao.date_util import get_month_start_end_date
 from app.dao.inbound_sms_dao import delete_inbound_sms_created_more_than_a_week_ago
 from app.dao.invited_user_dao import delete_invitations_created_more_than_two_days_ago
-from app.dao.jobs_dao import dao_set_scheduled_jobs_to_pending, dao_get_jobs_older_than_limited_by
+from app.dao.jobs_dao import (
+    dao_set_scheduled_jobs_to_pending,
+    dao_get_jobs_older_than_limited_by
+)
+from app.dao.monthly_billing_dao import (
+    get_service_ids_that_need_sms_billing_populated,
+    create_or_update_monthly_billing_sms
+)
 from app.dao.notifications_dao import (
     dao_timeout_notifications,
     is_delivery_slow_for_provider,
@@ -281,3 +289,14 @@ def delete_dvla_response_files_older_than_seven_days():
     except SQLAlchemyError as e:
         current_app.logger.exception("Failed to delete dvla response files")
         raise
+
+
+@notify_celery.task(name="populate_monthly_billing")
+@statsd(namespace="tasks")
+def populate_monthly_billing():
+    # for every service with billable units this month update billing totals for yesterday
+    # this will overwrite the existing amount.
+    yesterday = datetime.utcnow() - timedelta(days=1)
+    start_date, end_date = get_month_start_end_date(yesterday)
+    services = get_service_ids_that_need_sms_billing_populated(start_date, end_date=end_date)
+    [create_or_update_monthly_billing_sms(service_id=s.service_id, billing_month=start_date) for s in services]

app/commands.py

@@ -5,7 +5,8 @@ from flask.ext.script import Command, Manager, Option
 
 
 from app import db
-from app.models import (PROVIDERS, Service, User, NotificationHistory)
+from app.dao.monthly_billing_dao import create_or_update_monthly_billing_sms, get_monthly_billing_sms
+from app.models import (PROVIDERS, User)
 from app.dao.services_dao import (
     delete_service_and_all_associated_db_objects,
     dao_fetch_all_services_by_user
@@ -146,3 +147,18 @@ class CustomDbScript(Command):
             print('Committed {} updates at {}'.format(len(result), datetime.utcnow()))
             db.session.commit()
             result = db.session.execute(subq_hist).fetchall()
+
+
+class PopulateMonthlyBilling(Command):
+        option_list = (
+            Option('-s', '-service-id', dest='service_id',
+                   help="Service id to populate monthly billing for"),
+            Option('-m', '-month', dest="month", help="Use for integer value for month, e.g. 7 for July"),
+            Option('-y', '-year', dest="year", help="Use for integer value for year, e.g. 2017")
+        )
+
+        def run(self, service_id, month, year):
+            create_or_update_monthly_billing_sms(service_id, datetime(int(year), int(month), 1))
+            results = get_monthly_billing_sms(service_id, datetime(int(year), int(month), 1))
+            print("Finished populating data for {} for service id {}".format(month, service_id))
+            print(results.monthly_totals)

app/dao/date_util.py

@@ -16,3 +16,16 @@ def get_april_fools(year):
     """
     return pytz.timezone('Europe/London').localize(datetime(year, 4, 1, 0, 0, 0)).astimezone(pytz.UTC).replace(
         tzinfo=None)
+
+
+def get_month_start_end_date(month_year):
+    """
+     This function return the start and date of the month_year as UTC,
+     :param month_year: the datetime to calculate the start and end date for that month
+     :return: start_date, end_date, month
+    """
+    import calendar
+    _, num_days = calendar.monthrange(month_year.year, month_year.month)
+    first_day = datetime(month_year.year, month_year.month, 1, 0, 0, 0)
+    last_day = datetime(month_year.year, month_year.month, num_days, 23, 59, 59, 99999)
+    return first_day, last_day

app/dao/monthly_billing_dao.py

@@ -0,0 +1,55 @@
+from datetime import datetime
+
+from app import db
+from app.dao.date_util import get_month_start_end_date
+from app.dao.notification_usage_dao import get_billing_data_for_month
+from app.models import MonthlyBilling, SMS_TYPE, NotificationHistory
+
+
+def get_service_ids_that_need_sms_billing_populated(start_date, end_date):
+    return db.session.query(
+        NotificationHistory.service_id
+    ).filter(
+        NotificationHistory.created_at >= start_date,
+        NotificationHistory.created_at <= end_date,
+        NotificationHistory.notification_type == SMS_TYPE,
+        NotificationHistory.billable_units != 0
+    ).distinct().all()
+
+
+def create_or_update_monthly_billing_sms(service_id, billing_month):
+    start_date, end_date = get_month_start_end_date(billing_month)
+    monthly = get_billing_data_for_month(service_id=service_id, start_date=start_date, end_date=end_date)
+    # update monthly
+    monthly_totals = _monthly_billing_data_to_json(monthly)
+    row = MonthlyBilling.query.filter_by(year=billing_month.year,
+                                         month=datetime.strftime(billing_month, "%B"),
+                                         notification_type='sms').first()
+    if row:
+        row.monthly_totals = monthly_totals
+    else:
+        row = MonthlyBilling(service_id=service_id,
+                             notification_type=SMS_TYPE,
+                             year=billing_month.year,
+                             month=datetime.strftime(billing_month, "%B"),
+                             monthly_totals=monthly_totals)
+    db.session.add(row)
+    db.session.commit()
+
+
+def get_monthly_billing_sms(service_id, billing_month):
+    monthly = MonthlyBilling.query.filter_by(service_id=service_id,
+                                             year=billing_month.year,
+                                             month=datetime.strftime(billing_month, "%B"),
+                                             notification_type=SMS_TYPE).first()
+    return monthly
+
+
+def _monthly_billing_data_to_json(monthly):
+    # total cost must take into account the free allowance.
+    # might be a good idea to capture free allowance in this table
+    return [{"billing_units": x.billing_units,
+             "rate_multiplier": x.rate_multiplier,
+             "international": x.international,
+             "rate": x.rate,
+             "total_cost": (x.billing_units * x.rate_multiplier) * x.rate} for x in monthly]

app/dao/notification_usage_dao.py

@@ -6,7 +6,7 @@ from sqlalchemy import func, case, cast
 from sqlalchemy import literal_column
 
 from app import db
-from app.dao.date_util import get_financial_year
+from app.dao.date_util import get_financial_year, get_month_start_end_date
 from app.models import (NotificationHistory,
                         Rate,
                         NOTIFICATION_STATUS_TYPES_BILLABLE,
@@ -35,6 +35,20 @@ def get_yearly_billing_data(service_id, year):
     return sum(result, [])
 
 
+@statsd(namespace="dao")
+def get_billing_data_for_month(service_id, start_date, end_date):
+    rates = get_rates_for_year(start_date, end_date, SMS_TYPE)
+    result = []
+    # so the start end date in the query are the valid from the rate, not the month - this is going to take some thought
+    for r, n in zip(rates, rates[1:]):
+        result.extend(sms_billing_data_per_month_query(r.rate, service_id, max(r.valid_from, start_date),
+                                                       min(n.valid_from, end_date)))
+    result.extend(
+        sms_billing_data_per_month_query(rates[-1].rate, service_id, max(rates[-1].valid_from, start_date), end_date))
+
+    return result
+
+
 @statsd(namespace="dao")
 def get_monthly_billing_data(service_id, year):
     start_date, end_date = get_financial_year(year)
@@ -128,12 +142,12 @@ def is_between(date, start_date, end_date):
 def sms_billing_data_per_month_query(rate, service_id, start_date, end_date):
     month = get_london_month_from_utc_column(NotificationHistory.created_at)
     result = db.session.query(
-        month,
-        func.sum(NotificationHistory.billable_units),
-        rate_multiplier(),
+        month.label('month'),
+        func.sum(NotificationHistory.billable_units).label('billing_units'),
+        rate_multiplier().label('rate_multiplier'),
         NotificationHistory.international,
         NotificationHistory.notification_type,
-        cast(rate, Float())
+        cast(rate, Float()).label('rate')
     ).filter(
         *billing_data_filter(SMS_TYPE, start_date, end_date, service_id)
     ).group_by(

app/delivery/send_to_providers.py

@@ -18,7 +18,7 @@ from app.dao.templates_dao import dao_get_template_by_id
 from app.models import SMS_TYPE, KEY_TYPE_TEST, BRANDING_ORG, EMAIL_TYPE, NOTIFICATION_TECHNICAL_FAILURE, \
     NOTIFICATION_SENT, NOTIFICATION_SENDING
 
-from app.celery.statistics_tasks import record_initial_job_statistics, create_initial_notification_statistic_tasks
+from app.celery.statistics_tasks import create_initial_notification_statistic_tasks
 
 
 def send_sms_to_provider(notification):

app/models.py

@@ -4,7 +4,6 @@ import datetime
 from flask import url_for, current_app
 
 from sqlalchemy.ext.associationproxy import association_proxy
-from sqlalchemy.ext.hybrid import hybrid_property
 from sqlalchemy.dialects.postgresql import (
     UUID,
     JSON
@@ -1246,3 +1245,29 @@ class LetterRateDetail(db.Model):
     letter_rate = db.relationship('LetterRate', backref='letter_rates')
     page_total = db.Column(db.Integer, nullable=False)
     rate = db.Column(db.Numeric(), nullable=False)
+
+
+class MonthlyBilling(db.Model):
+    __tablename__ = 'monthly_billing'
+
+    id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+    service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'), index=True, nullable=False)
+    service = db.relationship('Service', backref='monthly_billing')
+    month = db.Column(db.String, nullable=False)
+    year = db.Column(db.Float(asdecimal=False), nullable=False)
+    notification_type = db.Column(notification_types, nullable=False)
+    monthly_totals = db.Column(JSON, nullable=False)
+    updated_at = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow)
+
+    __table_args__ = (
+        UniqueConstraint('service_id', 'month', 'year', 'notification_type', name='uix_monthly_billing'),
+    )
+
+    def serialized(self):
+        return {
+            "month": self.month,
+            "year": self.year,
+            "service_id": str(self.service_id),
+            "notification_type": self.notification_type,
+            "monthly_totals": self.monthly_totals
+        }

app/service/rest.py

@@ -68,7 +68,6 @@ from app.schemas import (
     user_schema,
     permission_schema,
     notification_with_template_schema,
-    notification_with_personalisation_schema,
     notifications_filter_schema,
     detailed_service_schema
 )

application.py

@@ -16,6 +16,7 @@ manager.add_command('db', MigrateCommand)
 manager.add_command('create_provider_rate', commands.CreateProviderRateCommand)
 manager.add_command('purge_functional_test_data', commands.PurgeFunctionalTestDataCommand)
 manager.add_command('custom_db_script', commands.CustomDbScript)
+manager.add_command('populate_monthly_billing', commands.PopulateMonthlyBilling)
 
 
 @manager.command

migrations/versions/0110_monthly_billing.py

@@ -0,0 +1,38 @@
+"""empty message
+
+Revision ID: 0110_monthly_billing
+Revises: 0109_rem_old_noti_status
+Create Date: 2017-07-13 14:35:03.183659
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '0110_monthly_billing'
+down_revision = '0109_rem_old_noti_status'
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+
+def upgrade():
+
+    op.create_table('monthly_billing',
+                    sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False),
+                    sa.Column('service_id', postgresql.UUID(as_uuid=True), nullable=False),
+                    sa.Column('month', sa.String(), nullable=False),
+                    sa.Column('year', sa.Float(), nullable=False),
+                    sa.Column('notification_type',
+                              postgresql.ENUM('email', 'sms', 'letter', name='notification_type', create_type=False),
+                              nullable=False),
+                    sa.Column('monthly_totals', postgresql.JSON(), nullable=False),
+                    sa.Column('updated_at', sa.DateTime, nullable=False),
+                    sa.ForeignKeyConstraint(['service_id'], ['services.id'], ),
+                    sa.PrimaryKeyConstraint('id')
+                    )
+    op.create_index(op.f('ix_monthly_billing_service_id'), 'monthly_billing', ['service_id'], unique=False)
+    op.create_index(op.f('uix_monthly_billing'), 'monthly_billing', ['service_id', 'month', 'year', 'notification_type'], unique=True)
+
+
+def downgrade():
+    op.drop_table('monthly_billing')

tests/app/celery/test_scheduled_tasks.py

@@ -25,8 +25,8 @@ from app.celery.scheduled_tasks import (
     send_scheduled_notifications,
     switch_current_sms_provider_on_slow_delivery,
     timeout_job_statistics,
-    timeout_notifications
-)
+    timeout_notifications,
+    populate_monthly_billing)
 from app.clients.performance_platform.performance_platform_client import PerformancePlatformClient
 from app.dao.jobs_dao import dao_get_job_by_id
 from app.dao.notifications_dao import dao_get_scheduled_notifications
@@ -36,10 +36,10 @@ from app.dao.provider_details_dao import (
 )
 from app.models import (
     Service, Template,
-    SMS_TYPE, LETTER_TYPE
-)
+    SMS_TYPE, LETTER_TYPE,
+    MonthlyBilling)
 from app.utils import get_london_midnight_in_utc
-from tests.app.db import create_notification, create_service, create_template, create_job
+from tests.app.db import create_notification, create_service, create_template, create_job, create_rate
 from tests.app.conftest import (
     sample_job as create_sample_job,
     sample_notification_history as create_notification_history,
@@ -98,6 +98,8 @@ def test_should_have_decorated_tasks_functions():
         'remove_transformed_dvla_files'
     assert delete_dvla_response_files_older_than_seven_days.__wrapped__.__name__ == \
         'delete_dvla_response_files_older_than_seven_days'
+    assert populate_monthly_billing.__wrapped__.__name__ == \
+        'populate_monthly_billing'
 
 
 @pytest.fixture(scope='function')
@@ -607,3 +609,30 @@ def test_delete_dvla_response_files_older_than_seven_days_does_not_remove_files(
     delete_dvla_response_files_older_than_seven_days()
 
     remove_s3_mock.assert_not_called()
+
+
+@freeze_time("2017-07-12 02:00:00")
+def test_populate_monthly_billing(sample_template):
+    yesterday = datetime(2017, 7, 11, 13, 30)
+    create_rate(datetime(2016, 1, 1), 0.0123, 'sms')
+    create_notification(template=sample_template, status='delivered', created_at=yesterday)
+    create_notification(template=sample_template, status='delivered', created_at=yesterday - timedelta(days=1))
+    create_notification(template=sample_template, status='delivered', created_at=yesterday + timedelta(days=1))
+    # not included in billing
+    create_notification(template=sample_template, status='delivered', created_at=yesterday - timedelta(days=30))
+
+    assert len(MonthlyBilling.query.all()) == 0
+    populate_monthly_billing()
+
+    monthly_billing = MonthlyBilling.query.all()
+    assert len(monthly_billing) == 1
+    assert monthly_billing[0].service_id == sample_template.service_id
+    assert monthly_billing[0].year == 2017
+    assert monthly_billing[0].month == 'July'
+    assert monthly_billing[0].notification_type == 'sms'
+    assert len(monthly_billing[0].monthly_totals) == 1
+    assert sorted(monthly_billing[0].monthly_totals[0]) == sorted({'international': False,
+                                                                   'rate_multiplier': 1,
+                                                                   'billing_units': 3,
+                                                                   'rate': 0.0123,
+                                                                   'total_cost': 0.0369})

tests/app/dao/test_date_utils.py

@@ -1,4 +1,8 @@
-from app.dao.date_util import get_financial_year, get_april_fools
+from datetime import datetime
+
+import pytest
+
+from app.dao.date_util import get_financial_year, get_april_fools, get_month_start_end_date
 
 
 def test_get_financial_year():
@@ -11,3 +15,16 @@ def test_get_april_fools():
     april_fools = get_april_fools(2016)
     assert str(april_fools) == '2016-03-31 23:00:00'
     assert april_fools.tzinfo is None
+
+
+@pytest.mark.parametrize("month, year, expected_end",
+                         [(7, 2017, 31),
+                          (2, 2016, 29),
+                          (2, 2017, 28),
+                          (9, 2018, 30),
+                          (12, 2019, 31)])
+def test_get_month_start_end_date(month, year, expected_end):
+    month_year = datetime(year, month, 10, 13, 30, 00)
+    result = get_month_start_end_date(month_year)
+    assert result[0] == datetime(year, month, 1, 0, 0, 0, 0)
+    assert result[1] == datetime(year, month, expected_end, 23, 59, 59, 99999)

tests/app/dao/test_monthly_billing.py

@@ -0,0 +1,126 @@
+from datetime import datetime
+
+from app.dao.monthly_billing_dao import (
+    create_or_update_monthly_billing_sms,
+    get_monthly_billing_sms,
+    get_service_ids_that_need_sms_billing_populated
+)
+from app.models import MonthlyBilling
+from tests.app.db import create_notification, create_rate, create_service, create_template
+
+
+def test_add_monthly_billing(sample_template):
+    jan = datetime(2017, 1, 1)
+    feb = datetime(2017, 2, 15)
+    create_rate(start_date=jan, value=0.0158, notification_type='sms')
+    create_notification(template=sample_template, created_at=jan, billable_units=1, status='delivered')
+    create_notification(template=sample_template, created_at=feb, billable_units=2, status='delivered')
+
+    create_or_update_monthly_billing_sms(service_id=sample_template.service_id,
+                                         billing_month=jan)
+    create_or_update_monthly_billing_sms(service_id=sample_template.service_id,
+                                         billing_month=feb)
+    monthly_billing = MonthlyBilling.query.all()
+    assert len(monthly_billing) == 2
+    assert monthly_billing[0].month == 'January'
+    assert monthly_billing[1].month == 'February'
+
+    january = get_monthly_billing_sms(service_id=sample_template.service_id, billing_month=jan)
+    expected_jan = {"billing_units": 1,
+                    "rate_multiplier": 1,
+                    "international": False,
+                    "rate": 0.0158,
+                    "total_cost": 1 * 0.0158}
+    assert_monthly_billing(january, 2017, "January", sample_template.service_id, 1, expected_jan)
+
+    february = get_monthly_billing_sms(service_id=sample_template.service_id, billing_month=feb)
+    expected_feb = {"billing_units": 2,
+                    "rate_multiplier": 1,
+                    "international": False,
+                    "rate": 0.0158,
+                    "total_cost": 2 * 0.0158}
+    assert_monthly_billing(february, 2017, "February", sample_template.service_id, 1, expected_feb)
+
+
+def test_add_monthly_billing_multiple_rates_in_a_month(sample_template):
+    rate_1 = datetime(2016, 12, 1)
+    rate_2 = datetime(2017, 1, 15)
+    create_rate(start_date=rate_1, value=0.0158, notification_type='sms')
+    create_rate(start_date=rate_2, value=0.0124, notification_type='sms')
+
+    create_notification(template=sample_template, created_at=datetime(2017, 1, 1), billable_units=1, status='delivered')
+    create_notification(template=sample_template, created_at=datetime(2017, 1, 14, 23, 59), billable_units=1,
+                        status='delivered')
+
+    create_notification(template=sample_template, created_at=datetime(2017, 1, 15), billable_units=2,
+                        status='delivered')
+    create_notification(template=sample_template, created_at=datetime(2017, 1, 17, 13, 30, 57), billable_units=4,
+                        status='delivered')
+
+    create_or_update_monthly_billing_sms(service_id=sample_template.service_id,
+                                         billing_month=rate_2)
+    monthly_billing = MonthlyBilling.query.all()
+    assert len(monthly_billing) == 1
+    assert monthly_billing[0].month == 'January'
+
+    january = get_monthly_billing_sms(service_id=sample_template.service_id, billing_month=rate_2)
+    first_row = {"billing_units": 2,
+                 "rate_multiplier": 1,
+                 "international": False,
+                 "rate": 0.0158,
+                 "total_cost": 3 * 0.0158}
+    assert_monthly_billing(january, 2017, "January", sample_template.service_id, 2, first_row)
+    second_row = {"billing_units": 6,
+                  "rate_multiplier": 1,
+                  "international": False,
+                  "rate": 0.0124,
+                  "total_cost": 1 * 0.0124}
+    assert sorted(january.monthly_totals[1]) == sorted(second_row)
+
+
+def test_update_monthly_billing_overwrites_old_totals(sample_template):
+    july = datetime(2017, 7, 1)
+    create_rate(july, 0.123, 'sms')
+    create_notification(template=sample_template, created_at=datetime(2017, 7, 2), billable_units=1, status='delivered')
+
+    create_or_update_monthly_billing_sms(sample_template.service_id, july)
+    first_update = get_monthly_billing_sms(sample_template.service_id, july)
+    expected = {"billing_units": 1,
+                "rate_multiplier": 1,
+                "international": False,
+                "rate": 0.123,
+                "total_cost": 1 * 0.123}
+    assert_monthly_billing(first_update, 2017, "July", sample_template.service_id, 1, expected)
+
+    create_notification(template=sample_template, created_at=datetime(2017, 7, 5), billable_units=2, status='delivered')
+    create_or_update_monthly_billing_sms(sample_template.service_id, july)
+    second_update = get_monthly_billing_sms(sample_template.service_id, july)
+    expected_update = {"billing_units": 3,
+                       "rate_multiplier": 1,
+                       "international": False,
+                       "rate": 0.123,
+                       "total_cost": 3 * 0.123}
+    assert_monthly_billing(second_update, 2017, "July", sample_template.service_id, 1, expected_update)
+
+
+def assert_monthly_billing(monthly_billing, year, month, service_id, expected_len, first_row):
+    assert monthly_billing.year == year
+    assert monthly_billing.month == month
+    assert monthly_billing.service_id == service_id
+    assert len(monthly_billing.monthly_totals) == expected_len
+    assert sorted(monthly_billing.monthly_totals[0]) == sorted(first_row)
+
+
+def test_get_service_id(notify_db_session):
+    service_1 = create_service(service_name="Service One")
+    template_1 = create_template(service=service_1)
+    service_2 = create_service(service_name="Service Two")
+    template_2 = create_template(service=service_2)
+    create_notification(template=template_1, created_at=datetime(2017, 6, 30, 13, 30), status='delivered')
+    create_notification(template=template_1, created_at=datetime(2017, 7, 1, 14, 30), status='delivered')
+    create_notification(template=template_2, created_at=datetime(2017, 7, 15, 13, 30))
+    create_notification(template=template_2, created_at=datetime(2017, 7, 31, 13, 30))
+    services = get_service_ids_that_need_sms_billing_populated(start_date=datetime(2017, 7, 1),
+                                                               end_date=datetime(2017, 7, 16))
+    expected_services = [service_1.id, service_2.id]
+    assert sorted([x.service_id for x in services]) == sorted(expected_services)

tests/app/dao/test_provider_rates_dao.py

@@ -1,3 +1,4 @@
+import uuid
 from datetime import datetime
 from decimal import Decimal
 from app.dao.provider_rates_dao import create_provider_rates

tests/app/db.py

@@ -1,7 +1,7 @@
 from datetime import datetime
 import uuid
 
-
+from app import db
 from app.dao.jobs_dao import dao_create_job
 from app.dao.service_inbound_api_dao import save_service_inbound_api
 from app.models import (
@@ -11,6 +11,7 @@ from app.models import (
     Notification,
     ScheduledNotification,
     ServicePermission,
+    Rate,
     Job,
     InboundSms,
     Organisation,
@@ -239,3 +240,10 @@ def create_organisation(colour='blue', logo='test_x2.png', name='test_org_1'):
     dao_create_organisation(organisation)
 
     return organisation
+
+
+def create_rate(start_date, value, notification_type):
+    rate = Rate(id=uuid.uuid4(), valid_from=start_date, rate=value, notification_type=notification_type)
+    db.session.add(rate)
+    db.session.commit()
+    return rate