Refactor the get_midnight functions to return the date in UTC but for the localised time.

So in June when we are in BST June 16, 00:00 BST => June 15 23:00 UTC

app/dao/notifications_dao.py

@@ -6,7 +6,7 @@ from datetime import (
 
 from flask import current_app
 from werkzeug.datastructures import MultiDict
-from sqlalchemy import (desc, func, or_, and_, asc, extract)
+from sqlalchemy import (desc, func, or_, and_, asc)
 from sqlalchemy.orm import joinedload
 
 from app import db, create_uuid
@@ -29,9 +29,8 @@ from app.models import (
 from app.dao.dao_utils import transactional
 from app.statsd_decorators import statsd
 from app.utils import (
-    get_midnight_for_date,
-    get_midnight_for_day_before
-)
+    get_midnight_for_day_before,
+    get_london_midnight_in_utc)
 
 
 def dao_get_notification_statistics_for_service_and_day(service_id, day):
@@ -432,7 +431,7 @@ def get_total_sent_notifications_in_date_range(start_date, end_date, notificatio
 def get_total_sent_notifications_yesterday():
     today = datetime.utcnow()
     start_date = get_midnight_for_day_before(today)
-    end_date = get_midnight_for_date(today)
+    end_date = get_london_midnight_in_utc(today)
 
     return {
         "start_date": start_date,

app/service/rest.py

@@ -53,7 +53,7 @@ from app.schemas import (
     notifications_filter_schema,
     detailed_service_schema
 )
-from app.utils import pagination_links
+from app.utils import pagination_links, get_london_midnight_in_utc
 
 service_blueprint = Blueprint('service', __name__)
 register_errors(service_blueprint)
@@ -281,8 +281,9 @@ def get_detailed_services(start_date, end_date, only_active=False, include_from_
     if start_date == datetime.utcnow().date():
         stats = dao_fetch_todays_stats_for_all_services(include_from_test_key=include_from_test_key)
     else:
-        stats = fetch_stats_by_date_range_for_all_services(start_date=start_date,
-                                                           end_date=end_date,
+
+        stats = fetch_stats_by_date_range_for_all_services(start_date=get_london_midnight_in_utc(start_date),
+                                                           end_date=get_london_midnight_in_utc(end_date),
                                                            include_from_test_key=include_from_test_key)
 
     for service_id, rows in itertools.groupby(stats, lambda x: x.service_id):

app/utils.py

@@ -1,5 +1,6 @@
 from datetime import datetime, timedelta
 
+import pytz
 from flask import url_for
 from app.models import SMS_TYPE, EMAIL_TYPE
 from notifications_utils.template import SMSMessageTemplate, PlainTextEmailTemplate
@@ -30,11 +31,18 @@ def get_template_instance(template, values):
     }[template['template_type']](template, values)
 
 
-def get_midnight_for_date(date):
-    midnight = datetime.combine(date, datetime.min.time())
-    return midnight
+def get_london_midnight_in_utc(date):
+    """
+     This function converts date to midnight as BST (British Standard Time) to UTC,
+     the tzinfo is lastly removed from the datetime because the database stores the timestamps without timezone.
+     :param date: the day to calculate the London midnight in UTC for
+     :return: the datetime of London midnight in UTC, for example 2016-06-17 = 2016-06-17 23:00:00
+    """
+    return pytz.timezone('Europe/London').localize(datetime.combine(date, datetime.min.time())).astimezone(
+        pytz.UTC).replace(
+        tzinfo=None)
 
 
 def get_midnight_for_day_before(date):
     day_before = date - timedelta(1)
-    return get_midnight_for_date(day_before)
+    return get_london_midnight_in_utc(day_before)

tests/app/celery/test_scheduled_tasks.py

@@ -16,7 +16,7 @@ from app.celery.scheduled_tasks import (
     send_daily_performance_stats
 )
 from app.dao.jobs_dao import dao_get_job_by_id
-from app.utils import get_midnight_for_date
+from app.utils import get_london_midnight_in_utc
 from tests.app.conftest import (
     sample_notification as create_sample_notification,
     sample_job as create_sample_job,
@@ -211,6 +211,6 @@ def test_send_daily_performance_stats_calls_with_correct_totals(notify_db, notif
     send_daily_performance_stats()
 
     perf_mock.assert_has_calls([
-        call(get_midnight_for_date(yesterday), 'sms', 2, 'day'),
-        call(get_midnight_for_date(yesterday), 'email', 3, 'day')
+        call(get_london_midnight_in_utc(yesterday), 'sms', 2, 'day'),
+        call(get_london_midnight_in_utc(yesterday), 'email', 3, 'day')
     ])

tests/app/dao/test_notification_dao.py

@@ -1,5 +1,4 @@
 from datetime import datetime, timedelta, date
-import pytz
 import uuid
 from functools import partial
 
@@ -47,7 +46,7 @@ from app.dao.notifications_dao import (
 from app.dao.services_dao import dao_update_service
 
 from app.utils import (
-    get_midnight_for_date,
+    get_london_midnight_in_utc,
     get_midnight_for_day_before
 )
 
@@ -1489,7 +1488,7 @@ def test_get_total_sent_notifications_yesterday_returns_expected_totals_dict(
 
     assert total_count_dict == {
         "start_date": get_midnight_for_day_before(datetime.utcnow()),
-        "end_date": get_midnight_for_date(datetime.utcnow()),
+        "end_date": get_london_midnight_in_utc(datetime.utcnow()),
         "email": {
             "count": 3
         },

tests/app/test_utils.py

@@ -2,24 +2,24 @@ from datetime import datetime
 import pytest
 
 from app.utils import (
-    get_midnight_for_date,
+    get_london_midnight_in_utc,
     get_midnight_for_day_before
 )
 
 
 @pytest.mark.parametrize('date, expected_date', [
     (datetime(2016, 1, 15, 0, 30), datetime(2016, 1, 15, 0, 0)),
-    (datetime(2016, 1, 15, 0, 0), datetime(2016, 1, 15, 0, 0)),
-    (datetime(2016, 1, 15, 11, 59), datetime(2016, 1, 15, 0, 0)),
+    (datetime(2016, 6, 15, 0, 0), datetime(2016, 6, 14, 23, 0)),
+    (datetime(2016, 9, 15, 11, 59), datetime(2016, 9, 14, 23, 0)),
 ])
-def test_get_midnight_for_today_returns_expected_date(date, expected_date):
-    assert get_midnight_for_date(date) == expected_date
+def test_get_london_midnight_in_utc_returns_expected_date(date, expected_date):
+    assert get_london_midnight_in_utc(date) == expected_date
 
 
 @pytest.mark.parametrize('date, expected_date', [
     (datetime(2016, 1, 15, 0, 30), datetime(2016, 1, 14, 0, 0)),
-    (datetime(2016, 1, 15, 0, 0), datetime(2016, 1, 14, 0, 0)),
-    (datetime(2016, 1, 15, 11, 59), datetime(2016, 1, 14, 0, 0)),
+    (datetime(2016, 7, 15, 0, 0), datetime(2016, 7, 13, 23, 0)),
+    (datetime(2016, 8, 23, 11, 59), datetime(2016, 8, 21, 23, 0)),
 ])
 def test_get_midnight_for_day_before_returns_expected_date(date, expected_date):
     assert get_midnight_for_day_before(date) == expected_date