darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-07-24 18:09:41 -04:00
Code Issues Packages Projects Releases Wiki Activity

notify-260 remove server-side timezone handling

Browse Source
This commit is contained in:
Kenneth Kehl
2023-05-10 08:39:50 -07:00
parent 2caeaa5be3
commit 08c1ad75c8
31 changed files with 143 additions and 175 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
app/celery/nightly_tasks.py
View File

@@ -1,7 +1,6 @@
from datetime import datetime, timedelta
from flask import current_app
from notifications_utils.timezones import convert_utc_to_local_timezone
from sqlalchemy.exc import SQLAlchemyError
from app import notify_celery
@@ -25,7 +24,7 @@ from app.dao.service_data_retention_dao import (
fetch_service_data_retention_for_all_services_by_notification_type,
)
from app.models import EMAIL_TYPE, SMS_TYPE, FactProcessingTime
from app.utils import get_local_midnight_in_utc
from app.utils import get_midnight_in_utc
@notify_celery.task(name="remove_sms_email_jobs")
@@ -64,9 +63,8 @@ def _delete_notifications_older_than_retention_by_type(notification_type):
flexible_data_retention = fetch_service_data_retention_for_all_services_by_notification_type(notification_type)
for f in flexible_data_retention:
day_to_delete_backwards_from = get_local_midnight_in_utc(
convert_utc_to_local_timezone(datetime.utcnow()).date() - timedelta(days=f.days_of_retention)
)
day_to_delete_backwards_from = get_midnight_in_utc(datetime.utcnow()).date() \
- timedelta(days=f.days_of_retention)
delete_notifications_for_service_and_type.apply_async(queue=QueueNames.REPORTING, kwargs={
'service_id': f.service_id,
@@ -74,9 +72,8 @@ def _delete_notifications_older_than_retention_by_type(notification_type):
'datetime_to_delete_before': day_to_delete_backwards_from
})
seven_days_ago = get_local_midnight_in_utc(
convert_utc_to_local_timezone(datetime.utcnow()).date() - timedelta(days=7)
)
seven_days_ago = get_midnight_in_utc(datetime.utcnow()).date() - timedelta(days=7)
service_ids_with_data_retention = {x.service_id for x in flexible_data_retention}
# get a list of all service ids that we'll need to delete for. Typically that might only be 5% of services.
@@ -168,8 +165,8 @@ def save_daily_notification_processing_time(local_date=None):
else:
local_date = datetime.strptime(local_date, "%Y-%m-%d").date()
start_time = get_local_midnight_in_utc(local_date)
end_time = get_local_midnight_in_utc(local_date + timedelta(days=1))
start_time = get_midnight_in_utc(local_date)
end_time = get_midnight_in_utc(local_date + timedelta(days=1))
result = dao_get_notifications_processing_time_stats(start_time, end_time)
insert_update_processing_time(
FactProcessingTime(

5
app/celery/reporting_tasks.py
View File

@@ -1,7 +1,6 @@
from datetime import datetime, timedelta
from flask import current_app
from notifications_utils.timezones import convert_utc_to_local_timezone
from app import notify_celery
from app.config import QueueNames
@@ -21,7 +20,7 @@ def create_nightly_billing(day_start=None):
# day_start is a datetime.date() object. e.g.
# up to 4 days of data counting back from day_start is consolidated
if day_start is None:
day_start = convert_utc_to_local_timezone(datetime.utcnow()).date() - timedelta(days=1)
day_start = datetime.utcnow().date() - timedelta(days=1)
else:
# When calling the task its a string in the format of "YYYY-MM-DD"
day_start = datetime.strptime(day_start, "%Y-%m-%d").date()
@@ -83,7 +82,7 @@ def create_nightly_notification_status():
mean the aggregated results are temporarily incorrect.
"""
yesterday = convert_utc_to_local_timezone(datetime.utcnow()).date() - timedelta(days=1)
yesterday = datetime.utcnow().date() - timedelta(days=1)
for notification_type in [SMS_TYPE, EMAIL_TYPE]:
days = 4

3
app/clients/performance_platform/performance_platform_client.py
View File

@@ -3,7 +3,6 @@ import json
import requests
from flask import current_app
from notifications_utils.timezones import convert_utc_to_local_timezone
class PerformancePlatformClient:
@@ -55,7 +54,7 @@ class PerformancePlatformClient:
:param period - the period that this data covers - "day", "week", "month", "quarter".
"""
payload = {
'_timestamp': convert_utc_to_local_timezone(start_time).isoformat(),
'_timestamp': start_time,
'service': 'govuk-notify',
'dataType': dataset,
'period': period,

6
app/commands.py
View File

@@ -62,7 +62,7 @@ from app.models import (
TemplateHistory,
User,
)
from app.utils import get_local_midnight_in_utc
from app.utils import get_midnight_in_utc
@click.group(name='command', help='Additional commands')
@@ -192,8 +192,8 @@ def rebuild_ft_billing_for_day(service_id, day):
rebuild_ft_data(day, service_id)
else:
services = get_service_ids_that_need_billing_populated(
get_local_midnight_in_utc(day),
get_local_midnight_in_utc(day + timedelta(days=1))
get_midnight_in_utc(day),
get_midnight_in_utc(day + timedelta(days=1))
)
for row in services:
rebuild_ft_data(day, row.service_id)

6
app/dao/complaint_dao.py
View File

@@ -6,7 +6,7 @@ from sqlalchemy import desc
from app import db
from app.dao.dao_utils import autocommit
from app.models import Complaint
from app.utils import get_local_midnight_in_utc
from app.utils import get_midnight_in_utc
@autocommit
@@ -28,7 +28,7 @@ def fetch_complaints_by_service(service_id):
def fetch_count_of_complaints(start_date, end_date):
start_date = get_local_midnight_in_utc(start_date)
end_date = get_local_midnight_in_utc(end_date + timedelta(days=1))
start_date = get_midnight_in_utc(start_date)
end_date = get_midnight_in_utc(end_date + timedelta(days=1))
return Complaint.query.filter(Complaint.created_at >= start_date, Complaint.created_at < end_date).count()

26
app/dao/date_util.py
View File

@@ -1,20 +1,13 @@
from datetime import date, datetime, time, timedelta
import pytz
from notifications_utils.timezones import (
convert_local_timezone_to_utc,
convert_utc_to_local_timezone,
local_timezone,
)
def get_months_for_financial_year(year):
return [
convert_local_timezone_to_utc(month) for month in (
month for month in (
get_months_for_year(4, 13, year)
+ get_months_for_year(1, 4, year + 1)
)
if convert_local_timezone_to_utc(month) < datetime.now()
if month < datetime.now()
]
@@ -30,8 +23,8 @@ def get_financial_year_dates(year):
year_start_datetime, year_end_datetime = get_financial_year(year)
return (
convert_utc_to_local_timezone(year_start_datetime).date(),
convert_utc_to_local_timezone(year_end_datetime).date()
year_start_datetime.date(),
year_end_datetime.date()
)
@@ -44,14 +37,7 @@ def get_current_financial_year():
def get_april_fools(year):
"""
This function converts the start of the financial year April 1, 00:00 as BST (British Standard Time) to UTC,
the tzinfo is lastly removed from the datetime because the database stores the timestamps without timezone.
:param year: the year to calculate the April 1, 00:00 BST for
:return: the datetime of April 1 for the given year, for example 2016 = 2016-03-31 23:00:00
"""
return local_timezone.localize(
datetime(year, 4, 1, 0, 0, 0)).astimezone(pytz.UTC).replace(tzinfo=None)
return datetime(year, 4, 1, 0, 0, 0)
def get_month_start_and_end_date_in_utc(month_year):
@@ -64,7 +50,7 @@ def get_month_start_and_end_date_in_utc(month_year):
_, num_days = calendar.monthrange(month_year.year, month_year.month)
first_day = datetime(month_year.year, month_year.month, 1, 0, 0, 0)
last_day = datetime(month_year.year, month_year.month, num_days, 23, 59, 59, 99999)
return convert_local_timezone_to_utc(first_day), convert_local_timezone_to_utc(last_day)
return first_day, last_day
def get_current_financial_year_start_year():

13
app/dao/fact_billing_dao.py
View File

@@ -1,7 +1,6 @@
from datetime import date, datetime, timedelta
from flask import current_app
from notifications_utils.timezones import convert_utc_to_local_timezone
from sqlalchemy import Date, Integer, and_, desc, func, union
from sqlalchemy.dialects.postgresql import insert
from sqlalchemy.sql.expression import case, literal
@@ -27,7 +26,7 @@ from app.models import (
Rate,
Service,
)
from app.utils import get_local_midnight_in_utc
from app.utils import get_midnight_in_utc
def fetch_sms_free_allowance_remainder_until_date(end_date):
@@ -179,7 +178,7 @@ def fetch_monthly_billing_for_year(service_id, year):
we also update the table on-the-fly if we need accurate data for this year.
"""
_, year_end = get_financial_year_dates(year)
today = convert_utc_to_local_timezone(datetime.utcnow()).date()
today = datetime.utcnow().date()
# if year end date is less than today, we are calculating for data in the past and have no need for deltas.
if year_end >= today:
@@ -330,8 +329,8 @@ def delete_billing_data_for_service_for_day(process_day, service_id):
def fetch_billing_data_for_day(process_day, service_id=None, check_permissions=False):
start_date = get_local_midnight_in_utc(process_day)
end_date = get_local_midnight_in_utc(process_day + timedelta(days=1))
start_date = get_midnight_in_utc(process_day)
end_date = get_midnight_in_utc(process_day + timedelta(days=1))
current_app.logger.info("Populate ft_billing for {} to {}".format(start_date, end_date))
transit_data = []
if not service_id:
@@ -430,7 +429,7 @@ def get_service_ids_that_need_billing_populated(start_date, end_date):
def get_rate(
rates, notification_type, date
):
start_of_day = get_local_midnight_in_utc(date)
start_of_day = get_midnight_in_utc(date)
if notification_type == SMS_TYPE:
return next(
@@ -624,7 +623,7 @@ def query_organisation_sms_usage_for_year(organisation_id, year):
def fetch_usage_year_for_organisation(organisation_id, year):
year_start, year_end = get_financial_year_dates(year)
today = convert_utc_to_local_timezone(datetime.utcnow()).date()
today = datetime.utcnow().date()
services = dao_get_organisation_live_services(organisation_id)
# if year end date is less than today, we are calculating for data in the past and have no need for deltas.

18
app/dao/fact_notification_status_dao.py
View File

@@ -28,16 +28,16 @@ from app.models import (
Template,
)
from app.utils import (
get_local_midnight_in_utc,
get_local_month_from_utc_column,
get_midnight_in_utc,
midnight_n_days_ago,
)
@autocommit
def update_fact_notification_status(process_day, notification_type, service_id):
start_date = get_local_midnight_in_utc(process_day)
end_date = get_local_midnight_in_utc(process_day + timedelta(days=1))
start_date = get_midnight_in_utc(process_day)
end_date = get_midnight_in_utc(process_day + timedelta(days=1))
# delete any existing rows in case some no longer exist e.g. if all messages are sent
FactNotificationStatus.query.filter(
@@ -112,8 +112,8 @@ def fetch_notification_status_for_service_for_day(bst_day, service_id):
Notification.status.label('notification_status'),
func.count().label('count')
).filter(
Notification.created_at >= get_local_midnight_in_utc(bst_day),
Notification.created_at < get_local_midnight_in_utc(bst_day + timedelta(days=1)),
Notification.created_at >= get_midnight_in_utc(bst_day),
Notification.created_at < get_midnight_in_utc(bst_day + timedelta(days=1)),
Notification.service_id == service_id,
Notification.key_type != KEY_TYPE_TEST
).group_by(
@@ -142,7 +142,7 @@ def fetch_notification_status_for_service_for_today_and_7_previous_days(service_
*([Notification.template_id] if by_template else []),
func.count().label('count')
).filter(
Notification.created_at >= get_local_midnight_in_utc(now),
Notification.created_at >= get_midnight_in_utc(now),
Notification.service_id == service_id,
Notification.key_type != KEY_TYPE_TEST
).group_by(
@@ -188,7 +188,7 @@ def fetch_notification_status_totals_for_all_services(start_date, end_date):
FactNotificationStatus.notification_status,
FactNotificationStatus.key_type,
)
today = get_local_midnight_in_utc(datetime.utcnow())
today = get_midnight_in_utc(datetime.utcnow())
if start_date <= datetime.utcnow().date() <= end_date:
stats_for_today = db.session.query(
Notification.notification_type.cast(db.Text).label('notification_type'),
@@ -265,7 +265,7 @@ def fetch_stats_for_all_services_by_date_range(start_date, end_date, include_fro
stats = stats.filter(FactNotificationStatus.key_type != KEY_TYPE_TEST)
if start_date <= datetime.utcnow().date() <= end_date:
today = get_local_midnight_in_utc(datetime.utcnow())
today = get_midnight_in_utc(datetime.utcnow())
subquery = db.session.query(
Notification.notification_type.cast(db.Text).label('notification_type'),
Notification.status.label('status'),
@@ -357,7 +357,7 @@ def fetch_monthly_template_usage_for_service(start_date, end_date, service_id):
)
if start_date <= datetime.utcnow() <= end_date:
today = get_local_midnight_in_utc(datetime.utcnow())
today = get_midnight_in_utc(datetime.utcnow())
month = get_local_month_from_utc_column(Notification.created_at)
stats_for_today = db.session.query(

6
app/dao/notifications_dao.py
View File

@@ -35,7 +35,7 @@ from app.models import (
)
from app.utils import (
escape_special_characters,
get_local_midnight_in_utc,
get_midnight_in_utc,
midnight_n_days_ago,
)
@@ -579,8 +579,8 @@ def get_service_ids_with_notifications_before(notification_type, timestamp):
def get_service_ids_with_notifications_on_date(notification_type, date):
start_date = get_local_midnight_in_utc(date)
end_date = get_local_midnight_in_utc(date + timedelta(days=1))
start_date = get_midnight_in_utc(date)
end_date = get_midnight_in_utc(date + timedelta(days=1))
notification_table_query = db.session.query(
Notification.service_id.label('service_id')

5
app/dao/provider_details_dao.py
View File

@@ -1,7 +1,6 @@
from datetime import datetime, timedelta
from flask import current_app
from notifications_utils.timezones import convert_utc_to_local_timezone
from sqlalchemy import asc, desc, func
from app import db
@@ -155,8 +154,8 @@ def _update_provider_details_without_commit(provider_details):
def dao_get_provider_stats():
# this query does not include the current day since the task to populate ft_billing runs overnight
current_local_datetime = convert_utc_to_local_timezone(datetime.utcnow())
first_day_of_the_month = current_local_datetime.date().replace(day=1)
current_datetime = datetime.utcnow()
first_day_of_the_month = current_datetime.date().replace(day=1)
subquery = db.session.query(
FactBilling.provider,

9
app/dao/services_dao.py
View File

@@ -42,7 +42,7 @@ from app.models import (
from app.utils import (
escape_special_characters,
get_archived_db_column_value,
get_local_midnight_in_utc,
get_midnight_in_utc,
)
DEFAULT_SERVICE_PERMISSIONS = [
@@ -388,8 +388,7 @@ def delete_service_and_all_associated_db_objects(service):
def dao_fetch_todays_stats_for_service(service_id):
today = date.today()
start_date = get_local_midnight_in_utc(today)
start_date = get_midnight_in_utc(today)
return db.session.query(
Notification.notification_type,
Notification.status,
@@ -406,8 +405,8 @@ def dao_fetch_todays_stats_for_service(service_id):
def dao_fetch_todays_stats_for_all_services(include_from_test_key=True, only_active=True):
today = date.today()
start_date = get_local_midnight_in_utc(today)
end_date = get_local_midnight_in_utc(today + timedelta(days=1))
start_date = get_midnight_in_utc(today)
end_date = get_midnight_in_utc(today + timedelta(days=1))
subquery = db.session.query(
Notification.notification_type,

4
app/models.py
View File

@@ -17,7 +17,6 @@ from notifications_utils.template import (
PlainTextEmailTemplate,
SMSMessageTemplate,
)
from notifications_utils.timezones import convert_utc_to_local_timezone
from sqlalchemy import CheckConstraint, Index, UniqueConstraint
from sqlalchemy.dialects.postgresql import JSON, JSONB, UUID
from sqlalchemy.ext.associationproxy import association_proxy
@@ -1503,7 +1502,6 @@ class Notification(db.Model):
return None
def serialize_for_csv(self):
created_at_in_est = convert_utc_to_local_timezone(self.created_at)
serialized = {
"row_number": '' if self.job_row_number is None else self.job_row_number + 1,
"recipient": self.to,
@@ -1512,7 +1510,7 @@ class Notification(db.Model):
"template_type": self.template.template_type,
"job_name": self.job.original_file_name if self.job else '',
"status": self.formatted_status,
"created_at": created_at_in_est.strftime("%Y-%m-%d %H:%M:%S"),
"created_at": self.created_at.strftime("%Y-%m-%d %H:%M:%S"),
"created_by_name": self.get_created_by_name(),
"created_by_email_address": self.get_created_by_email_address(),
}

6
app/performance_platform/processing_time.py
View File

@@ -6,12 +6,12 @@ from app import performance_platform_client
from app.dao.notifications_dao import (
dao_get_total_notifications_sent_per_day_for_performance_platform,
)
from app.utils import get_local_midnight_in_utc
from app.utils import get_midnight_in_utc
def send_processing_time_to_performance_platform(local_date):
start_time = get_local_midnight_in_utc(local_date)
end_time = get_local_midnight_in_utc(local_date + timedelta(days=1))
start_time = get_midnight_in_utc(local_date)
end_time = get_midnight_in_utc(local_date + timedelta(days=1))
send_processing_time_for_start_and_end(start_time, end_time, local_date)

6
app/platform_stats/rest.py
View File

@@ -17,7 +17,7 @@ from app.errors import InvalidRequest, register_errors
from app.platform_stats.platform_stats_schema import platform_stats_request
from app.schema_validation import validate
from app.service.statistics import format_admin_stats
from app.utils import get_local_midnight_in_utc
from app.utils import get_midnight_in_utc
platform_stats_blueprint = Blueprint('platform_stats', __name__)
@@ -54,8 +54,8 @@ def validate_date_range_is_within_a_financial_year(start_date, end_date):
if end_date < start_date:
raise InvalidRequest(message="Start date must be before end date", status_code=400)
start_fy = get_financial_year_for_datetime(get_local_midnight_in_utc(start_date))
end_fy = get_financial_year_for_datetime(get_local_midnight_in_utc(end_date))
start_fy = get_financial_year_for_datetime(get_midnight_in_utc(start_date))
end_fy = get_financial_year_for_datetime(get_midnight_in_utc(end_date))
if start_fy != end_fy:
raise InvalidRequest(message="Date must be in a single financial year.", status_code=400)

5
app/service/rest.py
View File

@@ -2,7 +2,6 @@ import itertools
from datetime import datetime
from flask import Blueprint, current_app, jsonify, request
from notifications_utils.timezones import convert_utc_to_local_timezone
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm.exc import NoResultFound
from werkzeug.datastructures import MultiDict
@@ -500,9 +499,7 @@ def get_monthly_notification_stats(service_id):
now = datetime.utcnow()
if end_date > now:
todays_deltas = fetch_notification_status_for_service_for_day(
convert_utc_to_local_timezone(now), service_id=service_id
)
todays_deltas = fetch_notification_status_for_service_for_day(now, service_id=service_id)
statistics.add_monthly_notification_status_stats(data, todays_deltas)
return jsonify(data=data)

4
app/service/statistics.py
View File

@@ -1,8 +1,6 @@
from collections import defaultdict
from datetime import datetime
from notifications_utils.timezones import convert_utc_to_local_timezone
from app.dao.date_util import get_months_for_financial_year
from app.models import NOTIFICATION_STATUS_TYPES, NOTIFICATION_TYPES
@@ -98,7 +96,7 @@ def create_empty_monthly_notification_status_stats_dict(year):
utc_month_starts = get_months_for_financial_year(year)
# nested dicts - data[month][template type][status] = count
return {
convert_utc_to_local_timezone(start).strftime('%Y-%m'): {
start.strftime('%Y-%m'): {
template_type: defaultdict(int)
for template_type in NOTIFICATION_TYPES
}

11
app/utils.py
View File

@@ -3,7 +3,6 @@ from os import getenv
from flask import url_for
from notifications_utils.template import HTMLEmailTemplate, SMSMessageTemplate
from notifications_utils.timezones import convert_local_timezone_to_utc
from sqlalchemy import func
DATETIME_FORMAT_NO_TIMEZONE = "%Y-%m-%d %H:%M:%S.%f"
@@ -49,19 +48,19 @@ def get_template_instance(template, values):
}[template['template_type']](template, values)
def get_local_midnight_in_utc(date):
def get_midnight_in_utc(date):
"""
This function converts date from midnight in local time to UTC,
This function converts date to midnight in UTC,
removing the tzinfo from the datetime because the database stores the timestamps without timezone.
:param date: the day to calculate the local midnight in UTC for
:return: the datetime of local midnight in UTC, for example 2016-06-17 = 2016-06-16 23:00:00
"""
return convert_local_timezone_to_utc(datetime.combine(date, datetime.min.time()))
return datetime.combine(date, datetime.min.time())
def get_midnight_for_day_before(date):
day_before = date - timedelta(1)
return get_local_midnight_in_utc(day_before)
return get_midnight_in_utc(day_before)
def get_local_month_from_utc_column(column):
@@ -95,7 +94,7 @@ def midnight_n_days_ago(number_of_days):
"""
Returns midnight a number of days ago. Takes care of daylight savings etc.
"""
return get_local_midnight_in_utc(datetime.utcnow() - timedelta(days=number_of_days))
return get_midnight_in_utc(datetime.utcnow() - timedelta(days=number_of_days))
def escape_special_characters(string):