darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-09-11 10:28:55 -04:00
Code Issues Packages Projects Releases Wiki Activity

notify-260 remove server-side timezone handling

Browse Source
This commit is contained in:
Kenneth Kehl
2023-05-10 08:39:50 -07:00
parent 2caeaa5be3
commit 08c1ad75c8
31 changed files with 143 additions and 175 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/celery/nightly_tasks.py
+7 -10
View File
@@ -1,7 +1,6 @@
from datetime import datetime, timedelta
from flask import current_app
from notifications_utils.timezones import convert_utc_to_local_timezone
from sqlalchemy.exc import SQLAlchemyError
from app import notify_celery
@@ -25,7 +24,7 @@ from app.dao.service_data_retention_dao import (
fetch_service_data_retention_for_all_services_by_notification_type,
)
from app.models import EMAIL_TYPE, SMS_TYPE, FactProcessingTime
from app.utils import get_local_midnight_in_utc
from app.utils import get_midnight_in_utc
@notify_celery.task(name="remove_sms_email_jobs")
@@ -64,9 +63,8 @@ def _delete_notifications_older_than_retention_by_type(notification_type):
flexible_data_retention = fetch_service_data_retention_for_all_services_by_notification_type(notification_type)
for f in flexible_data_retention:
day_to_delete_backwards_from = get_local_midnight_in_utc(
convert_utc_to_local_timezone(datetime.utcnow()).date() - timedelta(days=f.days_of_retention)
)
day_to_delete_backwards_from = get_midnight_in_utc(datetime.utcnow()).date() \
- timedelta(days=f.days_of_retention)
delete_notifications_for_service_and_type.apply_async(queue=QueueNames.REPORTING, kwargs={
'service_id': f.service_id,
@@ -74,9 +72,8 @@ def _delete_notifications_older_than_retention_by_type(notification_type):
'datetime_to_delete_before': day_to_delete_backwards_from
})
seven_days_ago = get_local_midnight_in_utc(
convert_utc_to_local_timezone(datetime.utcnow()).date() - timedelta(days=7)
)
seven_days_ago = get_midnight_in_utc(datetime.utcnow()).date() - timedelta(days=7)
service_ids_with_data_retention = {x.service_id for x in flexible_data_retention}
# get a list of all service ids that we'll need to delete for. Typically that might only be 5% of services.
@@ -168,8 +165,8 @@ def save_daily_notification_processing_time(local_date=None):
else:
local_date = datetime.strptime(local_date, "%Y-%m-%d").date()
start_time = get_local_midnight_in_utc(local_date)
end_time = get_local_midnight_in_utc(local_date + timedelta(days=1))
start_time = get_midnight_in_utc(local_date)
end_time = get_midnight_in_utc(local_date + timedelta(days=1))
result = dao_get_notifications_processing_time_stats(start_time, end_time)
insert_update_processing_time(
FactProcessingTime(
app/celery/reporting_tasks.py
+2 -3
View File
@@ -1,7 +1,6 @@
from datetime import datetime, timedelta
from flask import current_app
from notifications_utils.timezones import convert_utc_to_local_timezone
from app import notify_celery
from app.config import QueueNames
@@ -21,7 +20,7 @@ def create_nightly_billing(day_start=None):
# day_start is a datetime.date() object. e.g.
# up to 4 days of data counting back from day_start is consolidated
if day_start is None:
day_start = convert_utc_to_local_timezone(datetime.utcnow()).date() - timedelta(days=1)
day_start = datetime.utcnow().date() - timedelta(days=1)
else:
# When calling the task its a string in the format of "YYYY-MM-DD"
day_start = datetime.strptime(day_start, "%Y-%m-%d").date()
@@ -83,7 +82,7 @@ def create_nightly_notification_status():
mean the aggregated results are temporarily incorrect.
"""
yesterday = convert_utc_to_local_timezone(datetime.utcnow()).date() - timedelta(days=1)
yesterday = datetime.utcnow().date() - timedelta(days=1)
for notification_type in [SMS_TYPE, EMAIL_TYPE]:
days = 4
app/clients/performance_platform/performance_platform_client.py
+1 -2
View File
@@ -3,7 +3,6 @@ import json
import requests
from flask import current_app
from notifications_utils.timezones import convert_utc_to_local_timezone
class PerformancePlatformClient:
@@ -55,7 +54,7 @@ class PerformancePlatformClient:
:param period - the period that this data covers - "day", "week", "month", "quarter".
"""
payload = {
'_timestamp': convert_utc_to_local_timezone(start_time).isoformat(),
'_timestamp': start_time,
'service': 'govuk-notify',
'dataType': dataset,
'period': period,
app/commands.py
+3 -3
View File
@@ -62,7 +62,7 @@ from app.models import (
TemplateHistory,
User,
)
from app.utils import get_local_midnight_in_utc
from app.utils import get_midnight_in_utc
@click.group(name='command', help='Additional commands')
@@ -192,8 +192,8 @@ def rebuild_ft_billing_for_day(service_id, day):
rebuild_ft_data(day, service_id)
else:
services = get_service_ids_that_need_billing_populated(
get_local_midnight_in_utc(day),
get_local_midnight_in_utc(day + timedelta(days=1))
get_midnight_in_utc(day),
get_midnight_in_utc(day + timedelta(days=1))
)
for row in services:
rebuild_ft_data(day, row.service_id)
app/dao/complaint_dao.py
+3 -3
View File
@@ -6,7 +6,7 @@ from sqlalchemy import desc
from app import db
from app.dao.dao_utils import autocommit
from app.models import Complaint
from app.utils import get_local_midnight_in_utc
from app.utils import get_midnight_in_utc
@autocommit
@@ -28,7 +28,7 @@ def fetch_complaints_by_service(service_id):
def fetch_count_of_complaints(start_date, end_date):
start_date = get_local_midnight_in_utc(start_date)
end_date = get_local_midnight_in_utc(end_date + timedelta(days=1))
start_date = get_midnight_in_utc(start_date)
end_date = get_midnight_in_utc(end_date + timedelta(days=1))
return Complaint.query.filter(Complaint.created_at >= start_date, Complaint.created_at < end_date).count()
app/dao/date_util.py
+6 -20
View File
@@ -1,20 +1,13 @@
from datetime import date, datetime, time, timedelta
import pytz
from notifications_utils.timezones import (
convert_local_timezone_to_utc,
convert_utc_to_local_timezone,
local_timezone,
)
def get_months_for_financial_year(year):
return [
convert_local_timezone_to_utc(month) for month in (
month for month in (
get_months_for_year(4, 13, year)
+ get_months_for_year(1, 4, year + 1)
)
if convert_local_timezone_to_utc(month) < datetime.now()
if month < datetime.now()
]
@@ -30,8 +23,8 @@ def get_financial_year_dates(year):
year_start_datetime, year_end_datetime = get_financial_year(year)
return (
convert_utc_to_local_timezone(year_start_datetime).date(),
convert_utc_to_local_timezone(year_end_datetime).date()
year_start_datetime.date(),
year_end_datetime.date()
)
@@ -44,14 +37,7 @@ def get_current_financial_year():
def get_april_fools(year):
"""
This function converts the start of the financial year April 1, 00:00 as BST (British Standard Time) to UTC,
the tzinfo is lastly removed from the datetime because the database stores the timestamps without timezone.
:param year: the year to calculate the April 1, 00:00 BST for
:return: the datetime of April 1 for the given year, for example 2016 = 2016-03-31 23:00:00
"""
return local_timezone.localize(
datetime(year, 4, 1, 0, 0, 0)).astimezone(pytz.UTC).replace(tzinfo=None)
return datetime(year, 4, 1, 0, 0, 0)
def get_month_start_and_end_date_in_utc(month_year):
@@ -64,7 +50,7 @@ def get_month_start_and_end_date_in_utc(month_year):
_, num_days = calendar.monthrange(month_year.year, month_year.month)
first_day = datetime(month_year.year, month_year.month, 1, 0, 0, 0)
last_day = datetime(month_year.year, month_year.month, num_days, 23, 59, 59, 99999)
return convert_local_timezone_to_utc(first_day), convert_local_timezone_to_utc(last_day)
return first_day, last_day
def get_current_financial_year_start_year():
app/dao/fact_billing_dao.py
+6 -7
View File
@@ -1,7 +1,6 @@
from datetime import date, datetime, timedelta
from flask import current_app
from notifications_utils.timezones import convert_utc_to_local_timezone
from sqlalchemy import Date, Integer, and_, desc, func, union
from sqlalchemy.dialects.postgresql import insert
from sqlalchemy.sql.expression import case, literal
@@ -27,7 +26,7 @@ from app.models import (
Rate,
Service,
)
from app.utils import get_local_midnight_in_utc
from app.utils import get_midnight_in_utc
def fetch_sms_free_allowance_remainder_until_date(end_date):
@@ -179,7 +178,7 @@ def fetch_monthly_billing_for_year(service_id, year):
we also update the table on-the-fly if we need accurate data for this year.
"""
_, year_end = get_financial_year_dates(year)
today = convert_utc_to_local_timezone(datetime.utcnow()).date()
today = datetime.utcnow().date()
# if year end date is less than today, we are calculating for data in the past and have no need for deltas.
if year_end >= today:
@@ -330,8 +329,8 @@ def delete_billing_data_for_service_for_day(process_day, service_id):
def fetch_billing_data_for_day(process_day, service_id=None, check_permissions=False):
start_date = get_local_midnight_in_utc(process_day)
end_date = get_local_midnight_in_utc(process_day + timedelta(days=1))
start_date = get_midnight_in_utc(process_day)
end_date = get_midnight_in_utc(process_day + timedelta(days=1))
current_app.logger.info("Populate ft_billing for {} to {}".format(start_date, end_date))
transit_data = []
if not service_id:
@@ -430,7 +429,7 @@ def get_service_ids_that_need_billing_populated(start_date, end_date):
def get_rate(
rates, notification_type, date
):
start_of_day = get_local_midnight_in_utc(date)
start_of_day = get_midnight_in_utc(date)
if notification_type == SMS_TYPE:
return next(
@@ -624,7 +623,7 @@ def query_organisation_sms_usage_for_year(organisation_id, year):
def fetch_usage_year_for_organisation(organisation_id, year):
year_start, year_end = get_financial_year_dates(year)
today = convert_utc_to_local_timezone(datetime.utcnow()).date()
today = datetime.utcnow().date()
services = dao_get_organisation_live_services(organisation_id)
# if year end date is less than today, we are calculating for data in the past and have no need for deltas.
app/dao/fact_notification_status_dao.py
+9 -9
View File
@@ -28,16 +28,16 @@ from app.models import (
Template,
)
from app.utils import (
get_local_midnight_in_utc,
get_local_month_from_utc_column,
get_midnight_in_utc,
midnight_n_days_ago,
)
@autocommit
def update_fact_notification_status(process_day, notification_type, service_id):
start_date = get_local_midnight_in_utc(process_day)
end_date = get_local_midnight_in_utc(process_day + timedelta(days=1))
start_date = get_midnight_in_utc(process_day)
end_date = get_midnight_in_utc(process_day + timedelta(days=1))
# delete any existing rows in case some no longer exist e.g. if all messages are sent
FactNotificationStatus.query.filter(
@@ -112,8 +112,8 @@ def fetch_notification_status_for_service_for_day(bst_day, service_id):
Notification.status.label('notification_status'),
func.count().label('count')
).filter(
Notification.created_at >= get_local_midnight_in_utc(bst_day),
Notification.created_at < get_local_midnight_in_utc(bst_day + timedelta(days=1)),
Notification.created_at >= get_midnight_in_utc(bst_day),
Notification.created_at < get_midnight_in_utc(bst_day + timedelta(days=1)),
Notification.service_id == service_id,
Notification.key_type != KEY_TYPE_TEST
).group_by(
@@ -142,7 +142,7 @@ def fetch_notification_status_for_service_for_today_and_7_previous_days(service_
*([Notification.template_id] if by_template else []),
func.count().label('count')
).filter(
Notification.created_at >= get_local_midnight_in_utc(now),
Notification.created_at >= get_midnight_in_utc(now),
Notification.service_id == service_id,
Notification.key_type != KEY_TYPE_TEST
).group_by(
@@ -188,7 +188,7 @@ def fetch_notification_status_totals_for_all_services(start_date, end_date):
FactNotificationStatus.notification_status,
FactNotificationStatus.key_type,
)
today = get_local_midnight_in_utc(datetime.utcnow())
today = get_midnight_in_utc(datetime.utcnow())
if start_date <= datetime.utcnow().date() <= end_date:
stats_for_today = db.session.query(
Notification.notification_type.cast(db.Text).label('notification_type'),
@@ -265,7 +265,7 @@ def fetch_stats_for_all_services_by_date_range(start_date, end_date, include_fro
stats = stats.filter(FactNotificationStatus.key_type != KEY_TYPE_TEST)
if start_date <= datetime.utcnow().date() <= end_date:
today = get_local_midnight_in_utc(datetime.utcnow())
today = get_midnight_in_utc(datetime.utcnow())
subquery = db.session.query(
Notification.notification_type.cast(db.Text).label('notification_type'),
Notification.status.label('status'),
@@ -357,7 +357,7 @@ def fetch_monthly_template_usage_for_service(start_date, end_date, service_id):
)
if start_date <= datetime.utcnow() <= end_date:
today = get_local_midnight_in_utc(datetime.utcnow())
today = get_midnight_in_utc(datetime.utcnow())
month = get_local_month_from_utc_column(Notification.created_at)
stats_for_today = db.session.query(
app/dao/notifications_dao.py
+3 -3
View File
@@ -35,7 +35,7 @@ from app.models import (
)
from app.utils import (
escape_special_characters,
get_local_midnight_in_utc,
get_midnight_in_utc,
midnight_n_days_ago,
)
@@ -579,8 +579,8 @@ def get_service_ids_with_notifications_before(notification_type, timestamp):
def get_service_ids_with_notifications_on_date(notification_type, date):
start_date = get_local_midnight_in_utc(date)
end_date = get_local_midnight_in_utc(date + timedelta(days=1))
start_date = get_midnight_in_utc(date)
end_date = get_midnight_in_utc(date + timedelta(days=1))
notification_table_query = db.session.query(
Notification.service_id.label('service_id')
app/dao/provider_details_dao.py
+2 -3
View File
@@ -1,7 +1,6 @@
from datetime import datetime, timedelta
from flask import current_app
from notifications_utils.timezones import convert_utc_to_local_timezone
from sqlalchemy import asc, desc, func
from app import db
@@ -155,8 +154,8 @@ def _update_provider_details_without_commit(provider_details):
def dao_get_provider_stats():
# this query does not include the current day since the task to populate ft_billing runs overnight
current_local_datetime = convert_utc_to_local_timezone(datetime.utcnow())
first_day_of_the_month = current_local_datetime.date().replace(day=1)
current_datetime = datetime.utcnow()
first_day_of_the_month = current_datetime.date().replace(day=1)
subquery = db.session.query(
FactBilling.provider,
app/dao/services_dao.py
+4 -5
View File
@@ -42,7 +42,7 @@ from app.models import (
from app.utils import (
escape_special_characters,
get_archived_db_column_value,
get_local_midnight_in_utc,
get_midnight_in_utc,
)
DEFAULT_SERVICE_PERMISSIONS = [
@@ -388,8 +388,7 @@ def delete_service_and_all_associated_db_objects(service):
def dao_fetch_todays_stats_for_service(service_id):
today = date.today()
start_date = get_local_midnight_in_utc(today)
start_date = get_midnight_in_utc(today)
return db.session.query(
Notification.notification_type,
Notification.status,
@@ -406,8 +405,8 @@ def dao_fetch_todays_stats_for_service(service_id):
def dao_fetch_todays_stats_for_all_services(include_from_test_key=True, only_active=True):
today = date.today()
start_date = get_local_midnight_in_utc(today)
end_date = get_local_midnight_in_utc(today + timedelta(days=1))
start_date = get_midnight_in_utc(today)
end_date = get_midnight_in_utc(today + timedelta(days=1))
subquery = db.session.query(
Notification.notification_type,
app/models.py
+1 -3
View File
@@ -17,7 +17,6 @@ from notifications_utils.template import (
PlainTextEmailTemplate,
SMSMessageTemplate,
)
from notifications_utils.timezones import convert_utc_to_local_timezone
from sqlalchemy import CheckConstraint, Index, UniqueConstraint
from sqlalchemy.dialects.postgresql import JSON, JSONB, UUID
from sqlalchemy.ext.associationproxy import association_proxy
@@ -1503,7 +1502,6 @@ class Notification(db.Model):
return None
def serialize_for_csv(self):
created_at_in_est = convert_utc_to_local_timezone(self.created_at)
serialized = {
"row_number": '' if self.job_row_number is None else self.job_row_number + 1,
"recipient": self.to,
@@ -1512,7 +1510,7 @@ class Notification(db.Model):
"template_type": self.template.template_type,
"job_name": self.job.original_file_name if self.job else '',
"status": self.formatted_status,
"created_at": created_at_in_est.strftime("%Y-%m-%d %H:%M:%S"),
"created_at": self.created_at.strftime("%Y-%m-%d %H:%M:%S"),
"created_by_name": self.get_created_by_name(),
"created_by_email_address": self.get_created_by_email_address(),
}
app/performance_platform/processing_time.py
+3 -3
View File
@@ -6,12 +6,12 @@ from app import performance_platform_client
from app.dao.notifications_dao import (
dao_get_total_notifications_sent_per_day_for_performance_platform,
)
from app.utils import get_local_midnight_in_utc
from app.utils import get_midnight_in_utc
def send_processing_time_to_performance_platform(local_date):
start_time = get_local_midnight_in_utc(local_date)
end_time = get_local_midnight_in_utc(local_date + timedelta(days=1))
start_time = get_midnight_in_utc(local_date)
end_time = get_midnight_in_utc(local_date + timedelta(days=1))
send_processing_time_for_start_and_end(start_time, end_time, local_date)
app/platform_stats/rest.py
+3 -3
View File
@@ -17,7 +17,7 @@ from app.errors import InvalidRequest, register_errors
from app.platform_stats.platform_stats_schema import platform_stats_request
from app.schema_validation import validate
from app.service.statistics import format_admin_stats
from app.utils import get_local_midnight_in_utc
from app.utils import get_midnight_in_utc
platform_stats_blueprint = Blueprint('platform_stats', __name__)
@@ -54,8 +54,8 @@ def validate_date_range_is_within_a_financial_year(start_date, end_date):
if end_date < start_date:
raise InvalidRequest(message="Start date must be before end date", status_code=400)
start_fy = get_financial_year_for_datetime(get_local_midnight_in_utc(start_date))
end_fy = get_financial_year_for_datetime(get_local_midnight_in_utc(end_date))
start_fy = get_financial_year_for_datetime(get_midnight_in_utc(start_date))
end_fy = get_financial_year_for_datetime(get_midnight_in_utc(end_date))
if start_fy != end_fy:
raise InvalidRequest(message="Date must be in a single financial year.", status_code=400)
app/service/rest.py
+1 -4
View File
@@ -2,7 +2,6 @@ import itertools
from datetime import datetime
from flask import Blueprint, current_app, jsonify, request
from notifications_utils.timezones import convert_utc_to_local_timezone
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm.exc import NoResultFound
from werkzeug.datastructures import MultiDict
@@ -500,9 +499,7 @@ def get_monthly_notification_stats(service_id):
now = datetime.utcnow()
if end_date > now:
todays_deltas = fetch_notification_status_for_service_for_day(
convert_utc_to_local_timezone(now), service_id=service_id
)
todays_deltas = fetch_notification_status_for_service_for_day(now, service_id=service_id)
statistics.add_monthly_notification_status_stats(data, todays_deltas)
return jsonify(data=data)
app/service/statistics.py
+1 -3
View File
@@ -1,8 +1,6 @@
from collections import defaultdict
from datetime import datetime
from notifications_utils.timezones import convert_utc_to_local_timezone
from app.dao.date_util import get_months_for_financial_year
from app.models import NOTIFICATION_STATUS_TYPES, NOTIFICATION_TYPES
@@ -98,7 +96,7 @@ def create_empty_monthly_notification_status_stats_dict(year):
utc_month_starts = get_months_for_financial_year(year)
# nested dicts - data[month][template type][status] = count
return {
convert_utc_to_local_timezone(start).strftime('%Y-%m'): {
start.strftime('%Y-%m'): {
template_type: defaultdict(int)
for template_type in NOTIFICATION_TYPES
}
app/utils.py
+5 -6
View File
@@ -3,7 +3,6 @@ from os import getenv
from flask import url_for
from notifications_utils.template import HTMLEmailTemplate, SMSMessageTemplate
from notifications_utils.timezones import convert_local_timezone_to_utc
from sqlalchemy import func
DATETIME_FORMAT_NO_TIMEZONE = "%Y-%m-%d %H:%M:%S.%f"
@@ -49,19 +48,19 @@ def get_template_instance(template, values):
}[template['template_type']](template, values)
def get_local_midnight_in_utc(date):
def get_midnight_in_utc(date):
"""
This function converts date from midnight in local time to UTC,
This function converts date to midnight in UTC,
removing the tzinfo from the datetime because the database stores the timestamps without timezone.
:param date: the day to calculate the local midnight in UTC for
:return: the datetime of local midnight in UTC, for example 2016-06-17 = 2016-06-16 23:00:00
"""
return convert_local_timezone_to_utc(datetime.combine(date, datetime.min.time()))
return datetime.combine(date, datetime.min.time())
def get_midnight_for_day_before(date):
day_before = date - timedelta(1)
return get_local_midnight_in_utc(day_before)
return get_midnight_in_utc(day_before)
def get_local_month_from_utc_column(column):
@@ -95,7 +94,7 @@ def midnight_n_days_ago(number_of_days):
"""
Returns midnight a number of days ago. Takes care of daylight savings etc.
"""
return get_local_midnight_in_utc(datetime.utcnow() - timedelta(days=number_of_days))
return get_midnight_in_utc(datetime.utcnow() - timedelta(days=number_of_days))
def escape_special_characters(string):
tests/app/celery/test_nightly_tasks.py
+11 -11
View File
@@ -239,11 +239,11 @@ def test_delete_notifications_task_calls_task_for_services_with_data_retention_o
'service_id': sms_service.id,
'notification_type': 'sms',
# three days of retention, its morn of 5th, so we want to keep all messages from 4th, 3rd and 2nd.
'datetime_to_delete_before': datetime(2021, 6, 2, 4, 0),
'datetime_to_delete_before': date(2021, 6, 2),
})
@freeze_time('2021-04-05 03:00')
@freeze_time('2021-04-04 23:00')
def test_delete_notifications_task_calls_task_for_services_with_data_retention_by_looking_at_retention(
notify_db_session,
mocker
@@ -262,17 +262,17 @@ def test_delete_notifications_task_calls_task_for_services_with_data_retention_b
call(queue=ANY, kwargs={
'service_id': service_14_days.id,
'notification_type': 'sms',
'datetime_to_delete_before': datetime(2021, 3, 21, 4, 0)
'datetime_to_delete_before': date(2021, 3, 21)
}),
call(queue=ANY, kwargs={
'service_id': service_3_days.id,
'notification_type': 'sms',
'datetime_to_delete_before': datetime(2021, 4, 1, 4, 0)
'datetime_to_delete_before': date(2021, 4, 1)
}),
])
@freeze_time('2021-04-03 03:00')
@freeze_time('2021-04-02 23:00')
def test_delete_notifications_task_calls_task_for_services_that_have_sent_notifications_recently(
notify_db_session,
mocker
@@ -288,13 +288,13 @@ def test_delete_notifications_task_calls_task_for_services_that_have_sent_notifi
nothing_to_delete_email_template = create_template(service_nothing_to_delete, template_type='email')
# will be deleted as service has no custom retention, but past our default 7 days
create_notification(service_will_delete_1.templates[0], created_at=datetime.now() - timedelta(days=8))
create_notification(service_will_delete_2.templates[0], created_at=datetime.now() - timedelta(days=8))
create_notification(service_will_delete_1.templates[0], created_at=datetime.utcnow() - timedelta(days=8))
create_notification(service_will_delete_2.templates[0], created_at=datetime.utcnow() - timedelta(days=8))
# will be kept as it's recent, and we won't run delete_notifications_for_service_and_type
create_notification(nothing_to_delete_sms_template, created_at=datetime.now() - timedelta(days=2))
create_notification(nothing_to_delete_sms_template, created_at=datetime.utcnow() - timedelta(days=2))
# this is an old notification, but for email not sms, so we won't run delete_notifications_for_service_and_type
create_notification(nothing_to_delete_email_template, created_at=datetime.now() - timedelta(days=8))
create_notification(nothing_to_delete_email_template, created_at=datetime.utcnow() - timedelta(days=8))
mock_subtask = mocker.patch('app.celery.nightly_tasks.delete_notifications_for_service_and_type')
@@ -305,11 +305,11 @@ def test_delete_notifications_task_calls_task_for_services_that_have_sent_notifi
call(queue=ANY, kwargs={
'service_id': service_will_delete_1.id,
'notification_type': 'sms',
'datetime_to_delete_before': datetime(2021, 3, 26, 4, 0)
'datetime_to_delete_before': date(2021, 3, 26)
}),
call(queue=ANY, kwargs={
'service_id': service_will_delete_2.id,
'notification_type': 'sms',
'datetime_to_delete_before': datetime(2021, 3, 26, 4, 0)
'datetime_to_delete_before': date(2021, 3, 26)
}),
])
tests/app/celery/test_reporting_tasks.py
+3 -3
View File
@@ -73,7 +73,7 @@ def test_create_nightly_notification_status_triggers_tasks(
mock_celery.assert_called_with(
kwargs={
'service_id': sample_service.id,
'process_day': '2019-07-30',
'process_day': '2019-07-31',
'notification_type': SMS_TYPE
},
queue=QueueNames.REPORTING
@@ -352,7 +352,7 @@ def test_create_nightly_billing_for_day_use_BST(
)
create_notification(
created_at=datetime(2018, 3, 26, 3, 59),
created_at=datetime(2018, 3, 25, 23, 59),
template=sample_template,
status='delivered',
rate_multiplier=1.0,
@@ -361,7 +361,7 @@ def test_create_nightly_billing_for_day_use_BST(
# too early
create_notification(
created_at=datetime(2018, 3, 25, 3, 59),
created_at=datetime(2018, 3, 24, 23, 59),
template=sample_template,
status='delivered',
rate_multiplier=1.0,
tests/app/dao/notification_dao/test_notification_dao.py
+1 -1
View File
@@ -1495,7 +1495,7 @@ def test_notifications_not_yet_sent_return_no_rows(sample_service, notification_
@pytest.mark.parametrize('created_at_utc,date_to_check,expected_count', [
# Clocks change on the 27th of March 2022, so the query needs to look at the
# time range 00:00 - 23:00 (UTC) thereafter.
('2022-03-27T00:30', date(2022, 3, 27), 0), # 27/03 00:30 GMT
('2022-03-27T00:30', date(2022, 3, 27), 1), # 27/03 00:30 GMT
('2022-03-27T22:30', date(2022, 3, 27), 1), # 27/03 23:30 BST
('2022-03-27T23:30', date(2022, 3, 27), 1), # 28/03 00:30 BST
('2022-03-26T23:30', date(2022, 3, 26), 1), # 26/03 23:30 GMT
tests/app/dao/test_complaint_dao.py
+1 -1
View File
@@ -117,7 +117,7 @@ def test_fetch_count_of_complaints(sample_email_notification):
count_of_complaints = fetch_count_of_complaints(start_date=datetime(2018, 6, 7),
end_date=datetime(2018, 6, 7))
assert count_of_complaints == 3
assert count_of_complaints == 5
def test_fetch_count_of_complaints_returns_zero(notify_db_session):
tests/app/dao/test_date_utils.py
+11 -12
View File
@@ -12,22 +12,22 @@ from app.dao.date_util import (
def test_get_financial_year():
start, end = get_financial_year(2000)
assert str(start) == '2000-04-01 05:00:00'
assert str(end) == '2001-04-01 04:59:59.999999'
assert str(start) == '2000-04-01 00:00:00'
assert str(end) == '2001-03-31 23:59:59.999999'
def test_get_april_fools():
april_fools = get_april_fools(2016)
assert str(april_fools) == '2016-04-01 04:00:00'
assert str(april_fools) == '2016-04-01 00:00:00'
assert april_fools.tzinfo is None
@pytest.mark.parametrize("month, year, expected_start, expected_end", [
(7, 2017, datetime(2017, 7, 1, 4, 00, 00), datetime(2017, 8, 1, 3, 59, 59, 99999)),
(2, 2016, datetime(2016, 2, 1, 5, 00, 00), datetime(2016, 3, 1, 4, 59, 59, 99999)),
(2, 2017, datetime(2017, 2, 1, 5, 00, 00), datetime(2017, 3, 1, 4, 59, 59, 99999)),
(9, 2018, datetime(2018, 9, 1, 4, 00, 00), datetime(2018, 10, 1, 3, 59, 59, 99999)),
(12, 2019, datetime(2019, 12, 1, 5, 00, 00), datetime(2020, 1, 1, 4, 59, 59, 99999))
(7, 2017, datetime(2017, 7, 1, 0, 00, 00), datetime(2017, 7, 31, 23, 59, 59, 99999)),
(2, 2016, datetime(2016, 2, 1, 0, 00, 00), datetime(2016, 2, 29, 23, 59, 59, 99999)),
(2, 2017, datetime(2017, 2, 1, 0, 00, 00), datetime(2017, 2, 28, 23, 59, 59, 99999)),
(9, 2018, datetime(2018, 9, 1, 0, 00, 00), datetime(2018, 9, 30, 23, 59, 59, 99999)),
(12, 2019, datetime(2019, 12, 1, 0, 00, 00), datetime(2019, 12, 31, 23, 59, 59, 99999))
])
def test_get_month_start_and_end_date_in_utc(month, year, expected_start, expected_end):
month_year = datetime(year, month, 10, 13, 30, 00)
@@ -37,11 +37,10 @@ def test_get_month_start_and_end_date_in_utc(month, year, expected_start, expect
@pytest.mark.parametrize("dt, fy", [
(datetime(2018, 4, 1, 4, 0, 0), 2018),
(datetime(2019, 4, 1, 3, 59, 59), 2018),
(datetime(2019, 4, 1, 4, 0, 0), 2019),
(datetime(2018, 4, 1, 1, 0, 0), 2018),
(datetime(2019, 3, 31, 23, 59, 59), 2018),
(date(2019, 3, 31), 2018),
(date(2019, 4, 2), 2019), # date() gives midnight UTC, which is the day before in ET
(date(2019, 4, 2), 2019),
])
def test_get_financial_year_for_datetime(dt, fy):
assert get_financial_year_for_datetime(dt) == fy
tests/app/dao/test_fact_billing_dao.py
+14 -15
View File
@@ -3,7 +3,6 @@ from decimal import Decimal
import pytest
from freezegun import freeze_time
from notifications_utils.timezones import convert_utc_to_local_timezone
from app import db
from app.dao.fact_billing_dao import (
@@ -74,7 +73,7 @@ def test_fetch_billing_data_for_today_includes_data_with_the_right_key_type(noti
for key_type in ['normal', 'test', 'team']:
create_notification(template=template, status='delivered', key_type=key_type)
today = convert_utc_to_local_timezone(datetime.utcnow())
today = datetime.utcnow()
results = fetch_billing_data_for_day(today.date())
assert len(results) == 1
assert results[0].notifications_sent == 2
@@ -87,7 +86,7 @@ def test_fetch_billing_data_for_day_only_calls_query_for_permission_type(notify_
sms_template = create_template(service=service, template_type="sms")
create_notification(template=email_template, status='delivered')
create_notification(template=sms_template, status='delivered')
today = convert_utc_to_local_timezone(datetime.utcnow())
today = datetime.utcnow()
results = fetch_billing_data_for_day(process_day=today.date(), check_permissions=True)
assert len(results) == 1
@@ -99,7 +98,7 @@ def test_fetch_billing_data_for_day_only_calls_query_for_all_channels(notify_db_
sms_template = create_template(service=service, template_type="sms")
create_notification(template=email_template, status='delivered')
create_notification(template=sms_template, status='delivered')
today = convert_utc_to_local_timezone(datetime.utcnow())
today = datetime.utcnow()
results = fetch_billing_data_for_day(process_day=today.date(), check_permissions=False)
assert len(results) == 2
@@ -115,10 +114,10 @@ def test_fetch_billing_data_for_today_includes_data_with_the_right_date(notify_d
create_notification(template=template, status='delivered', created_at=datetime(2018, 4, 1, 0, 23, 23))
create_notification(template=template, status='sending', created_at=process_day + timedelta(days=1))
day_under_test = convert_utc_to_local_timezone(process_day)
day_under_test = process_day
results = fetch_billing_data_for_day(day_under_test.date())
assert len(results) == 1
assert results[0].notifications_sent == 2
assert results[0].notifications_sent == 3
def test_fetch_billing_data_for_day_is_grouped_by_template_and_notification_type(notify_db_session):
@@ -128,7 +127,7 @@ def test_fetch_billing_data_for_day_is_grouped_by_template_and_notification_type
create_notification(template=email_template, status='delivered')
create_notification(template=sms_template, status='delivered')
today = convert_utc_to_local_timezone(datetime.utcnow())
today = datetime.utcnow()
results = fetch_billing_data_for_day(today.date())
assert len(results) == 2
assert results[0].notifications_sent == 1
@@ -143,7 +142,7 @@ def test_fetch_billing_data_for_day_is_grouped_by_service(notify_db_session):
create_notification(template=email_template, status='delivered')
create_notification(template=sms_template, status='delivered')
today = convert_utc_to_local_timezone(datetime.utcnow())
today = datetime.utcnow()
results = fetch_billing_data_for_day(today.date())
assert len(results) == 2
assert results[0].notifications_sent == 1
@@ -156,7 +155,7 @@ def test_fetch_billing_data_for_day_is_grouped_by_provider(notify_db_session):
create_notification(template=template, status='delivered', sent_by='sns')
create_notification(template=template, status='delivered', sent_by='sns')
today = convert_utc_to_local_timezone(datetime.utcnow())
today = datetime.utcnow()
results = fetch_billing_data_for_day(today.date())
assert len(results) == 1
assert results[0].notifications_sent == 2
@@ -169,7 +168,7 @@ def test_fetch_billing_data_for_day_is_grouped_by_rate_mulitplier(notify_db_sess
create_notification(template=template, status='delivered', rate_multiplier=1)
create_notification(template=template, status='delivered', rate_multiplier=2)
today = convert_utc_to_local_timezone(datetime.utcnow())
today = datetime.utcnow()
results = fetch_billing_data_for_day(today.date())
assert len(results) == 2
assert results[0].notifications_sent == 1
@@ -182,7 +181,7 @@ def test_fetch_billing_data_for_day_is_grouped_by_international(notify_db_sessio
create_notification(template=sms_template, status='delivered', international=True)
create_notification(template=sms_template, status='delivered', international=False)
today = convert_utc_to_local_timezone(datetime.utcnow())
today = datetime.utcnow()
results = fetch_billing_data_for_day(today.date())
assert len(results) == 2
assert all(result.notifications_sent == 1 for result in results)
@@ -198,7 +197,7 @@ def test_fetch_billing_data_for_day_is_grouped_by_notification_type(notify_db_se
create_notification(template=email_template, status='delivered')
create_notification(template=email_template, status='delivered')
today = convert_utc_to_local_timezone(datetime.utcnow())
today = datetime.utcnow()
results = fetch_billing_data_for_day(today.date())
assert len(results) == 2
notification_types = [x.notification_type for x in results]
@@ -206,7 +205,7 @@ def test_fetch_billing_data_for_day_is_grouped_by_notification_type(notify_db_se
def test_fetch_billing_data_for_day_returns_empty_list(notify_db_session):
today = convert_utc_to_local_timezone(datetime.utcnow())
today = datetime.utcnow()
results = fetch_billing_data_for_day(today.date())
assert results == []
@@ -239,7 +238,7 @@ def test_fetch_billing_data_for_day_returns_list_for_given_service(notify_db_ses
create_notification(template=template, status='delivered')
create_notification(template=template_2, status='delivered')
today = convert_utc_to_local_timezone(datetime.utcnow())
today = datetime.utcnow()
results = fetch_billing_data_for_day(process_day=today.date(), service_id=service.id)
assert len(results) == 1
assert results[0].service_id == service.id
@@ -252,7 +251,7 @@ def test_fetch_billing_data_for_day_bills_correctly_for_status(notify_db_session
for status in NOTIFICATION_STATUS_TYPES:
create_notification(template=sms_template, status=status)
create_notification(template=email_template, status=status)
today = convert_utc_to_local_timezone(datetime.utcnow())
today = datetime.utcnow()
results = fetch_billing_data_for_day(process_day=today.date(), service_id=service.id)
sms_results = [x for x in results if x.notification_type == 'sms']
tests/app/dao/test_inbound_sms_dao.py
+11 -11
View File
@@ -64,8 +64,8 @@ def test_get_all_inbound_sms_filters_on_service(notify_db_session):
def test_get_all_inbound_sms_filters_on_time(sample_service, notify_db_session):
create_inbound_sms(sample_service, created_at=datetime(2017, 8, 7, 3, 59)) # sunday evening
sms_two = create_inbound_sms(sample_service, created_at=datetime(2017, 8, 7, 4, 0)) # monday (7th) morning
create_inbound_sms(sample_service, created_at=datetime(2017, 8, 6, 23, 59)) # sunday evening
sms_two = create_inbound_sms(sample_service, created_at=datetime(2017, 8, 7, 0, 0)) # monday (7th) morning
with freeze_time('2017-08-14 12:00'):
res = dao_get_inbound_sms_for_service(sample_service.id, limit_days=7)
@@ -87,9 +87,9 @@ def test_count_inbound_sms_for_service(notify_db_session):
def test_count_inbound_sms_for_service_filters_messages_older_than_n_days(sample_service):
# test between evening sunday 2nd of june and morning of monday 3rd
create_inbound_sms(sample_service, created_at=datetime(2019, 6, 3, 3, 59))
create_inbound_sms(sample_service, created_at=datetime(2019, 6, 3, 3, 59))
create_inbound_sms(sample_service, created_at=datetime(2019, 6, 3, 4, 1))
create_inbound_sms(sample_service, created_at=datetime(2019, 6, 2, 23, 59))
create_inbound_sms(sample_service, created_at=datetime(2019, 6, 2, 23, 59))
create_inbound_sms(sample_service, created_at=datetime(2019, 6, 3, 0, 1))
with freeze_time('Monday 10th June 2019 12:00'):
assert dao_count_inbound_sms_for_service(sample_service.id, limit_days=7) == 1
@@ -108,10 +108,10 @@ def test_should_delete_inbound_sms_according_to_data_retention(notify_db_session
create_service_data_retention(short_retention_service, notification_type='email', days_of_retention=4)
dates = [
datetime(2017, 6, 5, 4, 00), # just before three days
datetime(2017, 6, 5, 3, 59), # older than three days
datetime(2017, 6, 1, 4, 00), # just before seven days
datetime(2017, 6, 1, 3, 59), # older than seven days
datetime(2017, 6, 5, 0, 00), # just before three days
datetime(2017, 6, 4, 23, 59), # older than three days
datetime(2017, 6, 1, 0, 00), # just before seven days
datetime(2017, 5, 31, 23, 59), # older than seven days
datetime(2017, 5, 1, 0, 0), # older than thirty days
]
@@ -340,9 +340,9 @@ def test_most_recent_inbound_sms_paginates_properly(notify_api, sample_service):
def test_most_recent_inbound_sms_only_returns_values_within_7_days(sample_service):
# just out of bounds
create_inbound_sms(sample_service, user_number='1', content='old', created_at=datetime(2017, 4, 3, 3, 59, 59))
create_inbound_sms(sample_service, user_number='1', content='old', created_at=datetime(2017, 4, 2, 23, 59, 59))
# just in bounds
create_inbound_sms(sample_service, user_number='2', content='new', created_at=datetime(2017, 4, 3, 4, 0, 0))
create_inbound_sms(sample_service, user_number='2', content='new', created_at=datetime(2017, 4, 3, 0, 0, 0))
with freeze_time('Monday 10th April 2017 12:00:00'):
res = dao_get_paginated_most_recent_inbound_sms_by_user_number_for_service(sample_service.id, limit_days=7, page=1) # noqa
tests/app/dao/test_jobs_dao.py
+2 -2
View File
@@ -136,8 +136,8 @@ def test_get_jobs_for_service_with_limit_days_param(sample_template):
@freeze_time('2017-06-10')
def test_get_jobs_for_service_with_limit_days_edge_case(sample_template):
one_job = create_job(sample_template)
just_after_midnight_job = create_job(sample_template, created_at=datetime(2017, 6, 3, 4, 0, 1))
just_before_midnight_job = create_job(sample_template, created_at=datetime(2017, 6, 3, 3, 59, 0))
just_after_midnight_job = create_job(sample_template, created_at=datetime(2017, 6, 3, 0, 0, 1))
just_before_midnight_job = create_job(sample_template, created_at=datetime(2017, 6, 2, 23, 59, 0))
jobs_limit_days = dao_get_jobs_by_service_id(one_job.service_id, limit_days=7).items
assert len(jobs_limit_days) == 2
tests/app/dao/test_services_dao.py
+2 -2
View File
@@ -777,7 +777,7 @@ def test_dao_fetch_todays_stats_for_service_only_includes_today(notify_db_sessio
stats = dao_fetch_todays_stats_for_service(template.service_id)
stats = {row.status: row.count for row in stats}
assert 'delivered' not in stats
assert stats['delivered'] == 1
assert stats['failed'] == 1
assert stats['created'] == 1
@@ -807,7 +807,7 @@ def test_dao_fetch_todays_stats_for_service_only_includes_today_when_clocks_spri
def test_dao_fetch_todays_stats_for_service_only_includes_today_during_bst(notify_db_session):
template = create_template(service=create_service())
with freeze_time('2021-03-29T03:59:59'):
with freeze_time('2021-03-28T23:59:59'):
# just before midnight BST -- not included
create_notification(template=template, to_field='1', status='permanent-failure')
with freeze_time('2021-03-29T04:00:01'):
tests/app/inbound_sms/test_rest.py
+2 -2
View File
@@ -97,8 +97,8 @@ def test_post_to_get_inbound_sms_allows_badly_formatted_number(admin_request, sa
@freeze_time('Monday 10th April 2017 12:00')
def test_post_to_get_most_recent_inbound_sms_for_service_limits_to_a_week(admin_request, sample_service):
create_inbound_sms(sample_service, created_at=datetime(2017, 4, 3, 3, 59))
returned_inbound = create_inbound_sms(sample_service, created_at=datetime(2017, 4, 3, 4, 30))
create_inbound_sms(sample_service, created_at=datetime(2017, 4, 2, 23, 59))
returned_inbound = create_inbound_sms(sample_service, created_at=datetime(2017, 4, 3, 0, 30))
sms = admin_request.post('inbound_sms.post_inbound_sms_for_service', service_id=sample_service.id, _data={})
tests/app/job/test_rest.py
+2 -2
View File
@@ -751,10 +751,10 @@ def test_get_all_notifications_for_job_returns_csv_format(admin_request, sample_
}
@freeze_time('2017-06-10 04:00')
@freeze_time('2017-06-10 00:00')
def test_get_jobs_should_retrieve_from_ft_notification_status_for_old_jobs(admin_request, sample_template):
# it's the 10th today, so 3 days should include all of 7th, 8th, 9th, and some of 10th.
just_three_days_ago = datetime(2017, 6, 7, 3, 59, 59)
just_three_days_ago = datetime(2017, 6, 6, 23, 59, 59)
not_quite_three_days_ago = just_three_days_ago + timedelta(seconds=1)
job_1 = create_job(sample_template, created_at=just_three_days_ago, processing_started=just_three_days_ago)
tests/app/service/test_utils.py
+1 -1
View File
@@ -4,7 +4,7 @@ from app.dao.date_util import get_current_financial_year_start_year
# see get_financial_year for conversion of financial years.
@freeze_time("2017-04-01 03:59:59.999999")
@freeze_time("2017-03-31 23:59:59.999999")
def test_get_current_financial_year_start_year_before_march():
current_fy = get_current_financial_year_start_year()
assert current_fy == 2016
tests/app/test_model.py
+3 -3
View File
@@ -124,14 +124,14 @@ def test_notification_for_csv_returns_formatted_status(
@freeze_time("2017-03-26 23:01:53.321312")
def test_notification_for_csv_returns_bst_correctly(sample_template):
def test_notification_for_csv_returns_utc_correctly(sample_template):
notification = create_notification(sample_template)
serialized = notification.serialize_for_csv()
assert serialized['created_at'] == '2017-03-26 19:01:53'
assert serialized['created_at'] == '2017-03-26 23:01:53'
def test_notification_personalisation_getter_returns_empty_dict_from_None():
def test_notification_personalisation_getter_returns_empty_dict_from_none():
noti = Notification()
noti._personalisation = None
assert noti.personalisation == {}
tests/app/test_utils.py
+19 -19
View File
@@ -5,28 +5,28 @@ from freezegun import freeze_time
from app.utils import (
format_sequential_number,
get_local_midnight_in_utc,
get_midnight_for_day_before,
get_midnight_in_utc,
midnight_n_days_ago,
)
@pytest.mark.parametrize('date, expected_date', [
(datetime(2016, 1, 15, 0, 30), datetime(2016, 1, 15, 5, 0)),
(datetime(2016, 6, 15, 0, 0), datetime(2016, 6, 15, 4, 0)),
(datetime(2016, 9, 15, 11, 59), datetime(2016, 9, 15, 4, 0)),
(datetime(2016, 1, 15, 0, 30), datetime(2016, 1, 15, 0, 0)),
(datetime(2016, 6, 15, 0, 0), datetime(2016, 6, 15, 0, 0)),
(datetime(2016, 9, 15, 11, 59), datetime(2016, 9, 15, 0, 0)),
# works for both dates and datetimes
(date(2016, 1, 15), datetime(2016, 1, 15, 5, 0)),
(date(2016, 6, 15), datetime(2016, 6, 15, 4, 0)),
(date(2016, 1, 15), datetime(2016, 1, 15, 0, 0)),
(date(2016, 6, 15), datetime(2016, 6, 15, 0, 0)),
])
def test_get_local_midnight_in_utc_returns_expected_date(date, expected_date):
assert get_local_midnight_in_utc(date) == expected_date
def test_get_midnight_in_utc_returns_expected_date(date, expected_date):
assert get_midnight_in_utc(date) == expected_date
@pytest.mark.parametrize('date, expected_date', [
(datetime(2016, 1, 15, 0, 30), datetime(2016, 1, 14, 5, 0)),
(datetime(2016, 7, 15, 0, 0), datetime(2016, 7, 14, 4, 0)),
(datetime(2016, 8, 23, 11, 59), datetime(2016, 8, 22, 4, 0)),
(datetime(2016, 1, 15, 0, 30), datetime(2016, 1, 14, 0, 0)),
(datetime(2016, 7, 15, 0, 0), datetime(2016, 7, 14, 0, 0)),
(datetime(2016, 8, 23, 11, 59), datetime(2016, 8, 22, 0, 0)),
])
def test_get_midnight_for_day_before_returns_expected_date(date, expected_date):
assert get_midnight_for_day_before(date) == expected_date
@@ -34,20 +34,20 @@ def test_get_midnight_for_day_before_returns_expected_date(date, expected_date):
@pytest.mark.parametrize('current_time, arg, expected_datetime', [
# winter
('2018-01-10 23:59', 1, datetime(2018, 1, 9, 5, 0)),
('2018-01-11 00:00', 1, datetime(2018, 1, 10, 5, 0)),
('2018-01-10 23:59', 1, datetime(2018, 1, 9, 0, 0)),
('2018-01-11 00:00', 1, datetime(2018, 1, 10, 0, 0)),
# bst switchover at 1am 25th
('2018-03-25 10:00', 1, datetime(2018, 3, 24, 4, 0)),
('2018-03-26 10:00', 1, datetime(2018, 3, 25, 4, 0)),
('2018-03-27 10:00', 1, datetime(2018, 3, 26, 4, 0)),
('2018-03-25 10:00', 1, datetime(2018, 3, 24, 0, 0)),
('2018-03-26 10:00', 1, datetime(2018, 3, 25, 0, 0)),
('2018-03-27 10:00', 1, datetime(2018, 3, 26, 0, 0)),
# summer
('2018-06-05 10:00', 1, datetime(2018, 6, 4, 4, 0)),
('2018-06-05 10:00', 1, datetime(2018, 6, 4, 0, 0)),
# zero days ago
('2018-01-11 00:00', 0, datetime(2018, 1, 11, 5, 0)),
('2018-06-05 10:00', 0, datetime(2018, 6, 5, 4, 0)),
('2018-01-11 00:00', 0, datetime(2018, 1, 11, 0, 0)),
('2018-06-05 10:00', 0, datetime(2018, 6, 5, 0, 0)),
])
def test_midnight_n_days_ago(current_time, arg, expected_datetime):
with freeze_time(current_time):