Make billing year aware of British Summer Time

April 1st is in British summer time, ie 1hr ahead of UTC. The database
stores everything in UTC, so for accurate comparisions we need to make
sure that `get_financial_year()` returns a UTC, datetime-aware
timestamp that is 1hr ahead of midnight.

This also means that when we group notifications by month, the months
need to be in BST. So the line between one year and another is actually
01:00 on April 1st, _not_ 00:00 on April 1st.

There’s no way we’ve found to do this in SQLAlchemy or raw Postgres,
especially because we don’t store the timestamps with a timezone in the
database.

So the grouping and summing of the notifications has to be done in
Python.

app/dao/notifications_dao.py

@@ -1,9 +1,11 @@
 import uuid
+import pytz
 from datetime import (
     datetime,
     timedelta,
     date
 )
+from itertools import groupby
 
 from flask import current_app
 from werkzeug.datastructures import MultiDict
@@ -212,20 +214,25 @@ def get_notifications_for_job(service_id, job_id, filter_dict=None, page=1, page
 @statsd(namespace="dao")
 def get_notification_billable_unit_count_per_month(service_id, year):
     start, end = get_financial_year(year)
-    return db.session.query(
+
-        func.to_char(NotificationHistory.created_at, "FMMonth"),
+    notifications = db.session.query(
-        func.sum(NotificationHistory.billable_units)
+        NotificationHistory.created_at,
-    ).group_by(
+        NotificationHistory.billable_units
-        func.to_char(NotificationHistory.created_at, "FMMonth"),
-        func.to_char(NotificationHistory.created_at, "YYYY-MM")
     ).order_by(
-        func.to_char(NotificationHistory.created_at, "YYYY-MM")
+        NotificationHistory.created_at
     ).filter(
         NotificationHistory.service_id == service_id,
         NotificationHistory.created_at >= start,
         NotificationHistory.created_at < end
     ).all()
 
+    return [
+        (month, sum(count for _, count in row))
+        for month, row in groupby(
+            notifications, lambda row: get_bst_month(row[0])
+        )
+    ]
+
 
 @statsd(namespace="dao")
 def get_notification_with_personalisation(service_id, notification_id, key_type):
@@ -340,7 +347,17 @@ def dao_timeout_notifications(timeout_period_in_seconds):
 
 
 def get_financial_year(year):
-    return (
+    return (get_april_fools(year), get_april_fools(year + 1))
-        date(year, 4, 1),
+
-        date(year + 1, 4, 1)
+
-    )
+def get_april_fools(year):
+    return datetime(
+        year, 4, 1, 0, 0, 0, 0,
+        pytz.timezone("Europe/London")
+    ).astimezone(pytz.utc)
+
+
+def get_bst_month(datetime):
+    return pytz.utc.localize(datetime).replace(
+        tzinfo=pytz.timezone("Europe/London")
+    ).strftime('%B')

tests/app/dao/test_notification_dao.py

@@ -1,4 +1,5 @@
 from datetime import datetime, timedelta, date
+import pytz
 import uuid
 from functools import partial
 
@@ -690,18 +691,21 @@ def test_get_all_notifications_for_job_by_status(notify_db, notify_db_session, s
 def test_get_notification_billable_unit_count_per_month(notify_db, notify_db_session, sample_service):
 
     for year, month, day in (
-        (2017, 1, 1),
+        (2017, 1, 1),  # ↓ 2016 financial year
         (2016, 8, 1),
         (2016, 7, 31),
         (2016, 4, 6),
         (2016, 4, 6),
-        (2016, 4, 1),
+        (2016, 4, 1),  # ↓ 2015 financial year
         (2016, 3, 31),
         (2016, 1, 1)
     ):
         sample_notification(
             notify_db, notify_db_session, service=sample_service,
-            created_at=date(year, month, day)
+            created_at=datetime(
+                year, month, day, 0, 0, 0, 0,
+                tzinfo=pytz.utc
+            )
         )
 
     for financial_year, months in (
@@ -711,11 +715,11 @@ def test_get_notification_billable_unit_count_per_month(notify_db, notify_db_ses
         ),
         (
             2016,
-            [('April', 3), ('July', 1), ('August', 1), ('January', 1)]
+            [('April', 2), ('July', 1), ('August', 1), ('January', 1)]
         ),
         (
             2015,
-            [('January', 1), ('March', 1)]
+            [('January', 1), ('March', 1), ('April', 1)]
         ),
         (
             2014,
@@ -1204,5 +1208,7 @@ def test_should_exclude_test_key_notifications_by_default(
 
 def test_get_financial_year():
     start, end = get_financial_year(2000)
-    assert start == date(2000, 4, 1)
+    assert start.tzinfo == pytz.utc
-    assert end == date(2001, 4, 1)
+    assert start.isoformat() == '2000-04-01T00:01:00+00:00'
+    assert end.tzinfo == pytz.utc
+    assert end.isoformat() == '2001-04-01T00:01:00+00:00'