Created new queries to return the rate with the sum of billable units for the year totals.

Once we have the new columns in notifications table, the query will need to include the rate multiplier and if the number is international.
The monthly billing query will be built next.

app/dao/date_util.py

@@ -0,0 +1,18 @@
+from datetime import datetime
+
+import pytz
+
+
+def get_financial_year(year):
+    return get_april_fools(year), get_april_fools(year + 1)
+
+
+def get_april_fools(year):
+    """
+     This function converts the start of the financial year April 1, 00:00 as BST (British Standard Time) to UTC,
+     the tzinfo is lastly removed from the datetime becasue the database stores the timestamps without timezone.
+     :param year: the year to calculate the April 1, 00:00 BST for
+     :return: the datetime of April 1 for the given year, for example 2016 = 2016-03-31 23:00:00
+    """
+    return pytz.timezone('Europe/London').localize(datetime(year, 4, 1, 0, 0, 0)).astimezone(pytz.UTC).replace(
+        tzinfo=None)

app/dao/notification_usage_dao.py

@@ -0,0 +1,69 @@
+from decimal import Decimal
+
+from sqlalchemy import func
+
+from app import db
+from app.dao.date_util import get_financial_year
+from app.models import (NotificationHistory,
+                        Rate,
+                        NOTIFICATION_STATUS_TYPES_BILLABLE,
+                        KEY_TYPE_TEST,
+                        SMS_TYPE,
+                        EMAIL_TYPE)
+from app.statsd_decorators import statsd
+
+
+@statsd(namespace="dao")
+def get_yearly_billing_data(service_id, year):
+    start_date, end_date = get_financial_year(year)
+    rates = get_rates_for_year(start_date, end_date, SMS_TYPE)
+
+    result = []
+    for r, n in zip(rates, rates[1:]):
+        result.append(
+            sms_billing_data_query(str(r.rate), service_id, r.valid_from, n.valid_from))
+
+    result.append(sms_billing_data_query(str(rates[-1].rate), service_id, rates[-1].valid_from, end_date))
+
+    result.append(email_billing_data_query(service_id, start_date, end_date))
+
+    return result
+
+
+def billing_data_filter(notification_type, start_date, end_date, service_id):
+    return [NotificationHistory.notification_type == notification_type,
+            NotificationHistory.created_at >= start_date,
+            NotificationHistory.created_at < end_date,
+            NotificationHistory.service_id == service_id,
+            NotificationHistory.status.in_(NOTIFICATION_STATUS_TYPES_BILLABLE),
+            NotificationHistory.key_type != KEY_TYPE_TEST
+            ]
+
+
+def email_billing_data_query(service_id, start_date, end_date):
+    result = db.session.query(func.count(NotificationHistory.id),
+                              NotificationHistory.notification_type,
+                              "0"
+                              ).filter(*billing_data_filter(EMAIL_TYPE, start_date, end_date, service_id)
+                                       ).group_by(NotificationHistory.notification_type).first()
+    if not result:
+        return 0, EMAIL_TYPE, Decimal("0")
+    else:
+        return result
+
+
+def sms_billing_data_query(rate, service_id, start_date, end_date):
+    result = db.session.query(func.sum(NotificationHistory.billable_units),
+                              NotificationHistory.notification_type,
+                              rate
+                              ).filter(*billing_data_filter(SMS_TYPE, start_date, end_date, service_id)
+                                       ).group_by(NotificationHistory.notification_type).first()
+    if not result:
+        return 0, SMS_TYPE, Decimal("0")
+    else:
+        return result
+
+
+def get_rates_for_year(start_date, end_date, notification_type):
+    return Rate.query.filter(Rate.valid_from >= start_date, Rate.valid_from < end_date,
+                             Rate.notification_type == notification_type).order_by(Rate.valid_from).all()

app/dao/notifications_dao.py

@@ -1,5 +1,4 @@
 import functools
-import pytz
 from datetime import (
     datetime,
     timedelta,
@@ -12,6 +11,7 @@ from sqlalchemy.orm import joinedload
 
 from app import db, create_uuid
 from app.dao import days_ago
+from app.dao.date_util import get_financial_year
 from app.models import (
     Service,
     Notification,
@@ -243,13 +243,15 @@ def get_notifications_for_job(service_id, job_id, filter_dict=None, page=1, page
 def get_notification_billable_unit_count_per_month(service_id, year):
     month = get_london_month_from_utc_column(NotificationHistory.created_at)
 
+    start_date, end_date = get_financial_year(year)
     notifications = db.session.query(
         month,
         func.sum(NotificationHistory.billable_units)
     ).filter(
         NotificationHistory.billable_units != 0,
         NotificationHistory.service_id == service_id,
-        NotificationHistory.created_at.between(*get_financial_year(year)),
+        NotificationHistory.created_at >= start_date,
+        NotificationHistory.created_at < end_date
     ).group_by(
         month
     ).order_by(
@@ -410,21 +412,6 @@ def dao_timeout_notifications(timeout_period_in_seconds):
     return updated
 
 
-def get_financial_year(year):
-    return get_april_fools(year), get_april_fools(year + 1)
-
-
-def get_april_fools(year):
-    """
-     This function converts the start of the financial year April 1, 00:00 as BST (British Standard Time) to UTC,
-     the tzinfo is lastly removed from the datetime becasue the database stores the timestamps without timezone.
-     :param year: the year to calculate the April 1, 00:00 BST for
-     :return: the datetime of April 1 for the given year, for example 2016 = 2016-03-31 23:00:00
-    """
-    return pytz.timezone('Europe/London').localize(datetime(year, 4, 1, 0, 0, 0)).astimezone(pytz.UTC).replace(
-        tzinfo=None)
-
-
 def get_total_sent_notifications_in_date_range(start_date, end_date, notification_type):
     result = db.session.query(
         func.count(NotificationHistory.id).label('count')

app/dao/rates_dao.py

@@ -1,11 +0,0 @@
-from sqlalchemy import desc
-
-from app import db
-from app.models import Rate
-
-
-def get_rate_for_type_and_date(notification_type, date_sent):
-    return db.session.query(Rate).filter(Rate.notification_type == notification_type,
-                                         Rate.valid_from <= date_sent
-                                         ).order_by(Rate.valid_from.desc()
-                                                    ).limit(1).first()

app/dao/services_dao.py

@@ -229,6 +229,7 @@ def _stats_for_service_query(service_id):
 def dao_fetch_monthly_historical_stats_by_template_for_service(service_id, year):
     month = get_london_month_from_utc_column(NotificationHistory.created_at)
 
+    start_date, end_date = get_financial_year(year)
     sq = db.session.query(
         NotificationHistory.template_id,
         NotificationHistory.status,
@@ -236,7 +237,9 @@ def dao_fetch_monthly_historical_stats_by_template_for_service(service_id, year)
         func.count().label('count')
     ).filter(
         NotificationHistory.service_id == service_id,
-        NotificationHistory.created_at.between(*get_financial_year(year))
+        NotificationHistory.created_at >= start_date,
+        NotificationHistory.created_at < end_date
+
     ).group_by(
         month,
         NotificationHistory.template_id,
@@ -262,6 +265,7 @@ def dao_fetch_monthly_historical_stats_by_template_for_service(service_id, year)
 def dao_fetch_monthly_historical_stats_for_service(service_id, year):
     month = get_london_month_from_utc_column(NotificationHistory.created_at)
 
+    start_date, end_date = get_financial_year(year)
     rows = db.session.query(
         NotificationHistory.notification_type,
         NotificationHistory.status,
@@ -269,7 +273,8 @@ def dao_fetch_monthly_historical_stats_for_service(service_id, year):
         func.count(NotificationHistory.id).label('count')
     ).filter(
         NotificationHistory.service_id == service_id,
-        NotificationHistory.created_at.between(*get_financial_year(year)),
+        NotificationHistory.created_at >= start_date,
+        NotificationHistory.created_at < end_date
     ).group_by(
         NotificationHistory.notification_type,
         NotificationHistory.status,

tests/app/dao/test_date_utils.py

@@ -0,0 +1,13 @@
+from app.dao.date_util import get_financial_year, get_april_fools
+
+
+def test_get_financial_year():
+    start, end = get_financial_year(2000)
+    assert str(start) == '2000-03-31 23:00:00'
+    assert str(end) == '2001-03-31 23:00:00'
+
+
+def test_get_april_fools():
+    april_fools = get_april_fools(2016)
+    assert str(april_fools) == '2016-03-31 23:00:00'
+    assert april_fools.tzinfo is None

tests/app/dao/test_notification_dao.py

@@ -3,7 +3,6 @@ import uuid
 from functools import partial
 
 import pytest
-
 from freezegun import freeze_time
 from sqlalchemy.exc import SQLAlchemyError, IntegrityError
 
@@ -18,8 +17,7 @@ from app.models import (
     NOTIFICATION_SENT,
     KEY_TYPE_NORMAL,
     KEY_TYPE_TEAM,
-    KEY_TYPE_TEST
-)
+    KEY_TYPE_TEST)
 
 from app.dao.notifications_dao import (
     dao_create_notification,
@@ -40,8 +38,6 @@ from app.dao.notifications_dao import (
     update_notification_status_by_reference,
     dao_delete_notifications_and_history_by_id,
     dao_timeout_notifications,
-    get_financial_year,
-    get_april_fools,
     is_delivery_slow_for_provider,
     dao_update_notifications_sent_to_dvla)
 
@@ -1359,18 +1355,6 @@ def test_should_exclude_test_key_notifications_by_default(
     assert len(all_notifications) == 1
 
 
-def test_get_financial_year():
-    start, end = get_financial_year(2000)
-    assert str(start) == '2000-03-31 23:00:00'
-    assert str(end) == '2001-03-31 23:00:00'
-
-
-def test_get_april_fools():
-    april_fools = get_april_fools(2016)
-    assert str(april_fools) == '2016-03-31 23:00:00'
-    assert april_fools.tzinfo is None
-
-
 @pytest.mark.parametrize('notification_type', ['sms', 'email'])
 def test_get_total_sent_notifications_in_date_range_returns_only_in_date_range(
     notify_db,

tests/app/dao/test_notification_usage_dao.py

@@ -0,0 +1,97 @@
+import uuid
+from datetime import datetime
+
+from decimal import Decimal
+
+from app.dao.notification_usage_dao import (get_rates_for_year, get_yearly_billing_data)
+from app.models import Rate
+from tests.app.db import create_notification
+
+
+def test_get_rates_for_year(notify_db, notify_db_session):
+    set_up_rate(notify_db, datetime(2016, 4, 1), 1.50)
+    set_up_rate(notify_db, datetime(2017, 6, 1), 1.75)
+    rates = get_rates_for_year(datetime(2016, 4, 1), datetime(2017, 3, 31), 'sms')
+    assert len(rates) == 1
+    assert datetime.strftime(rates[0].valid_from, '%Y-%m-%d %H:%M:%S') == "2016-04-01 00:00:00"
+    assert rates[0].rate == Decimal("1.50")
+    rates = get_rates_for_year(datetime(2017, 4, 1), datetime(2018, 3, 31), 'sms')
+    assert len(rates) == 1
+    assert datetime.strftime(rates[0].valid_from, '%Y-%m-%d %H:%M:%S') == "2017-06-01 00:00:00"
+    assert rates[0].rate == Decimal("1.75")
+
+
+def test_get_yearly_billing_data(notify_db, notify_db_session, sample_template, sample_email_template):
+    set_up_rate(notify_db, datetime(2016, 4, 1), 1.40)
+    set_up_rate(notify_db, datetime(2016, 6, 1), 1.58)
+    # previous year
+    create_notification(template=sample_template, created_at=datetime(2016, 3, 31), sent_at=datetime(2016, 3, 31),
+                        status='sending', billable_units=1)
+    # current year
+    create_notification(template=sample_template, created_at=datetime(2016, 4, 2), sent_at=datetime(2016, 4, 2),
+                        status='sending', billable_units=1)
+    create_notification(template=sample_template, created_at=datetime(2016, 5, 18), sent_at=datetime(2016, 5, 18),
+                        status='sending', billable_units=2)
+    create_notification(template=sample_template, created_at=datetime(2016, 7, 22), sent_at=datetime(2016, 7, 22),
+                        status='sending', billable_units=3)
+    create_notification(template=sample_template, created_at=datetime(2016, 9, 15), sent_at=datetime(2016, 9, 15),
+                        status='sending', billable_units=4)
+    create_notification(template=sample_template, created_at=datetime(2017, 3, 31), sent_at=datetime(2017, 3, 31),
+                        status='sending', billable_units=5)
+    create_notification(template=sample_email_template, created_at=datetime(2016, 9, 15), sent_at=datetime(2016, 9, 15),
+                        status='sending', billable_units=0)
+    create_notification(template=sample_email_template, created_at=datetime(2017, 3, 31), sent_at=datetime(2017, 3, 31),
+                        status='sending', billable_units=0)
+    # next year
+    create_notification(template=sample_template, created_at=datetime(2017, 4, 1), sent_at=datetime(2017, 4, 1),
+                        status='sending', billable_units=6)
+    results = get_yearly_billing_data(sample_template.service_id, 2016)
+    assert len(results) == 3
+    assert results[0] == (3, 'sms', Decimal('1.4'))
+    assert results[1] == (12, 'sms', Decimal('1.58'))
+    assert results[2] == (2, 'email', Decimal("0"))
+
+
+def test_get_yearly_billing_data_with_one_rate(notify_db, notify_db_session, sample_template):
+    set_up_rate(notify_db, datetime(2016, 4, 1), 1.40)
+    # previous year
+    create_notification(template=sample_template, created_at=datetime(2016, 3, 31), sent_at=datetime(2016, 3, 31),
+                        status='sending', billable_units=1)
+    # current year
+    create_notification(template=sample_template, created_at=datetime(2016, 4, 2), sent_at=datetime(2016, 4, 2),
+                        status='sending', billable_units=1)
+    create_notification(template=sample_template, created_at=datetime(2016, 5, 18), sent_at=datetime(2016, 5, 18),
+                        status='sending', billable_units=2)
+    create_notification(template=sample_template, created_at=datetime(2016, 7, 22), sent_at=datetime(2016, 7, 22),
+                        status='sending', billable_units=3)
+    create_notification(template=sample_template, created_at=datetime(2016, 9, 15), sent_at=datetime(2016, 9, 15),
+                        status='sending', billable_units=4)
+    create_notification(template=sample_template, created_at=datetime(2017, 3, 31, 22, 59, 59),
+                        sent_at=datetime(2017, 3, 31), status='sending', billable_units=5)
+    # next year
+    create_notification(template=sample_template, created_at=datetime(2017, 3, 31, 23, 00, 00),
+                        sent_at=datetime(2017, 3, 31), status='sending', billable_units=6)
+    create_notification(template=sample_template, created_at=datetime(2017, 4, 1), sent_at=datetime(2017, 4, 1),
+                        status='sending', billable_units=7)
+    results = get_yearly_billing_data(sample_template.service_id, 2016)
+    assert len(results) == 2
+    assert results[0] == (15, 'sms', Decimal('1.4'))
+    assert results[1] == (0, 'email', Decimal('0'))
+
+
+def test_get_yearly_billing_data_with_no_sms_notifications(notify_db, notify_db_session, sample_email_template):
+    set_up_rate(notify_db, datetime(2016, 4, 1), 1.40)
+    create_notification(template=sample_email_template, created_at=datetime(2016, 7, 31), sent_at=datetime(2016, 3, 31),
+                        status='sending', billable_units=0)
+    create_notification(template=sample_email_template, created_at=datetime(2016, 10, 2), sent_at=datetime(2016, 4, 2),
+                        status='sending', billable_units=0)
+
+    results = get_yearly_billing_data(sample_email_template.service_id, 2016)
+    assert len(results) == 2
+    assert results[0] == (0, 'sms', Decimal('0'))
+    assert results[1] == (2, 'email', Decimal('0'))
+
+
+def set_up_rate(notify_db, start_date, value):
+    rate = Rate(id=uuid.uuid4(), valid_from=start_date, rate=value, notification_type='sms')
+    notify_db.session.add(rate)

tests/app/dao/test_rates_dao.py

@@ -1,18 +0,0 @@
-from datetime import datetime
-
-from decimal import Decimal
-
-from app.dao.rates_dao import get_rate_for_type_and_date
-
-
-def test_get_rate_for_type_and_date(notify_db):
-    rate = get_rate_for_type_and_date('sms', datetime.utcnow())
-    assert rate.rate == Decimal("1.58")
-
-    rate = get_rate_for_type_and_date('sms', datetime(2016, 6, 1))
-    assert rate.rate == Decimal("1.65")
-
-
-def test_get_rate_for_type_and_date_early_date(notify_db):
-    rate = get_rate_for_type_and_date('sms', datetime(2014, 6, 1))
-    assert not rate

tests/conftest.py

@@ -76,8 +76,7 @@ def notify_db_session(notify_db):
                             "job_status",
                             "provider_details_history",
                             "template_process_type",
-                            "dvla_organisation",
-                            "rates"]:
+                            "dvla_organisation"]:
             notify_db.engine.execute(tbl.delete())
     notify_db.session.commit()