Merge pull request #1022 from GSA/notify-api-951

replace utcnow with timezone naive utc call

app/dao/api_key_dao.py

@@ -1,11 +1,12 @@
 import uuid
-from datetime import datetime, timedelta
+from datetime import timedelta
 
 from sqlalchemy import func, or_
 
 from app import db
 from app.dao.dao_utils import autocommit, version_class
 from app.models import ApiKey
+from app.utils import utc_now
 
 
 @autocommit
@@ -23,7 +24,7 @@ def save_model_api_key(api_key):
 @version_class(ApiKey)
 def expire_api_key(service_id, api_key_id):
     api_key = ApiKey.query.filter_by(id=api_key_id, service_id=service_id).one()
-    api_key.expiry_date = datetime.utcnow()
+    api_key.expiry_date = utc_now()
     db.session.add(api_key)
 
 
@@ -32,7 +33,7 @@ def get_model_api_keys(service_id, id=None):
         return ApiKey.query.filter_by(
             id=id, service_id=service_id, expiry_date=None
         ).one()
-    seven_days_ago = datetime.utcnow() - timedelta(days=7)
+    seven_days_ago = utc_now() - timedelta(days=7)
     return ApiKey.query.filter(
         or_(
             ApiKey.expiry_date == None,  # noqa

app/dao/date_util.py

@@ -1,5 +1,7 @@
 from datetime import date, datetime, time, timedelta
 
+from app.utils import utc_now
+
 
 def get_months_for_financial_year(year):
     return [
@@ -22,7 +24,7 @@ def get_calendar_year_dates(year):
 
 
 def get_current_calendar_year():
-    now = datetime.utcnow()
+    now = utc_now()
     current_year = int(now.strftime("%Y"))
     year = current_year
     return get_calendar_year(year)

app/dao/fact_billing_dao.py

@@ -1,4 +1,4 @@
-from datetime import date, datetime, timedelta
+from datetime import date, timedelta
 
 from flask import current_app
 from sqlalchemy import Date, Integer, and_, desc, func, union
@@ -18,7 +18,7 @@ from app.models import (
     Rate,
     Service,
 )
-from app.utils import get_midnight_in_utc
+from app.utils import get_midnight_in_utc, utc_now
 
 
 def fetch_sms_free_allowance_remainder_until_date(end_date):
@@ -198,7 +198,7 @@ def fetch_monthly_billing_for_year(service_id, year):
     we also update the table on-the-fly if we need accurate data for this year.
     """
     _, year_end = get_calendar_year_dates(year)
-    today = datetime.utcnow().date()
+    today = utc_now().date()
 
     # if year end date is less than today, we are calculating for data in the past and have no need for deltas.
     if year_end >= today:
@@ -535,7 +535,7 @@ def update_fact_billing(data, process_day):
         set_={
             "notifications_sent": stmt.excluded.notifications_sent,
             "billable_units": stmt.excluded.billable_units,
-            "updated_at": datetime.utcnow(),
+            "updated_at": utc_now(),
         },
     )
     db.session.connection().execute(stmt)
@@ -699,7 +699,7 @@ def query_organization_sms_usage_for_year(organization_id, year):
 
 def fetch_usage_year_for_organization(organization_id, year):
     year_start, year_end = get_calendar_year_dates(year)
-    today = datetime.utcnow().date()
+    today = utc_now().date()
     services = dao_get_organization_live_services(organization_id)
 
     # if year end date is less than today, we are calculating for data in the past and have no need for deltas.

app/dao/fact_notification_status_dao.py

@@ -1,4 +1,4 @@
-from datetime import datetime, timedelta
+from datetime import timedelta
 
 from sqlalchemy import Date, case, func
 from sqlalchemy.dialects.postgresql import insert
@@ -19,6 +19,7 @@ from app.utils import (
     get_midnight_in_utc,
     get_month_from_utc_column,
     midnight_n_days_ago,
+    utc_now,
 )
 
 
@@ -128,7 +129,7 @@ def fetch_notification_status_for_service_for_today_and_7_previous_days(
     service_id, by_template=False, limit_days=7
 ):
     start_date = midnight_n_days_ago(limit_days)
-    now = datetime.utcnow()
+    now = utc_now()
     stats_for_7_days = db.session.query(
         FactNotificationStatus.notification_type.cast(db.Text).label(
             "notification_type"
@@ -212,8 +213,8 @@ def fetch_notification_status_totals_for_all_services(start_date, end_date):
             FactNotificationStatus.key_type,
         )
     )
-    today = get_midnight_in_utc(datetime.utcnow())
-    if start_date <= datetime.utcnow().date() <= end_date:
+    today = get_midnight_in_utc(utc_now())
+    if start_date <= utc_now().date() <= end_date:
         stats_for_today = (
             db.session.query(
                 Notification.notification_type.cast(db.Text).label("notification_type"),
@@ -299,8 +300,8 @@ def fetch_stats_for_all_services_by_date_range(
     if not include_from_test_key:
         stats = stats.filter(FactNotificationStatus.key_type != KeyType.TEST)
 
-    if start_date <= datetime.utcnow().date() <= end_date:
-        today = get_midnight_in_utc(datetime.utcnow())
+    if start_date <= utc_now().date() <= end_date:
+        today = get_midnight_in_utc(utc_now())
         subquery = (
             db.session.query(
                 Notification.notification_type.label("notification_type"),
@@ -395,8 +396,8 @@ def fetch_monthly_template_usage_for_service(start_date, end_date, service_id):
         )
     )
 
-    if start_date <= datetime.utcnow() <= end_date:
-        today = get_midnight_in_utc(datetime.utcnow())
+    if start_date <= utc_now() <= end_date:
+        today = get_midnight_in_utc(utc_now())
         month = get_month_from_utc_column(Notification.created_at)
 
         stats_for_today = (

app/dao/fact_processing_time_dao.py

@@ -1,11 +1,10 @@
-from datetime import datetime
-
 from sqlalchemy.dialects.postgresql import insert
 from sqlalchemy.sql.expression import case
 
 from app import db
 from app.dao.dao_utils import autocommit
 from app.models import FactProcessingTime
+from app.utils import utc_now
 
 
 @autocommit
@@ -27,7 +26,7 @@ def insert_update_processing_time(processing_time):
         set_={
             "messages_total": stmt.excluded.messages_total,
             "messages_within_10_secs": stmt.excluded.messages_within_10_secs,
-            "updated_at": datetime.utcnow(),
+            "updated_at": utc_now(),
         },
     )
     db.session.connection().execute(stmt)

app/dao/invited_org_user_dao.py

@@ -1,7 +1,8 @@
-from datetime import datetime, timedelta
+from datetime import timedelta
 
 from app import db
 from app.models import InvitedOrganizationUser
+from app.utils import utc_now
 
 
 def save_invited_org_user(invited_org_user):
@@ -28,9 +29,7 @@ def get_invited_org_users_for_organization(organization_id):
 def delete_org_invitations_created_more_than_two_days_ago():
     deleted = (
         db.session.query(InvitedOrganizationUser)
-        .filter(
-            InvitedOrganizationUser.created_at <= datetime.utcnow() - timedelta(days=2)
-        )
+        .filter(InvitedOrganizationUser.created_at <= utc_now() - timedelta(days=2))
         .delete()
     )
     db.session.commit()

app/dao/invited_user_dao.py

@@ -1,8 +1,9 @@
-from datetime import datetime, timedelta
+from datetime import timedelta
 
 from app import db
 from app.enums import InvitedUserStatus
 from app.models import InvitedUser
+from app.utils import utc_now
 
 
 def save_invited_user(invited_user):
@@ -41,7 +42,7 @@ def expire_invitations_created_more_than_two_days_ago():
     expired = (
         db.session.query(InvitedUser)
         .filter(
-            InvitedUser.created_at <= datetime.utcnow() - timedelta(days=2),
+            InvitedUser.created_at <= utc_now() - timedelta(days=2),
             InvitedUser.status.in_((InvitedUserStatus.PENDING,)),
         )
         .update({InvitedUser.status: InvitedUserStatus.EXPIRED})

app/dao/jobs_dao.py

@@ -1,5 +1,5 @@
 import uuid
-from datetime import datetime, timedelta
+from datetime import timedelta
 
 from flask import current_app
 from sqlalchemy import and_, asc, desc, func
@@ -13,7 +13,7 @@ from app.models import (
     ServiceDataRetention,
     Template,
 )
-from app.utils import midnight_n_days_ago
+from app.utils import midnight_n_days_ago, utc_now
 
 
 def dao_get_notification_outcomes_for_job(service_id, job_id):
@@ -110,7 +110,7 @@ def dao_set_scheduled_jobs_to_pending():
     jobs = (
         Job.query.filter(
             Job.job_status == JobStatus.SCHEDULED,
-            Job.scheduled_for < datetime.utcnow(),
+            Job.scheduled_for < utc_now(),
         )
         .order_by(asc(Job.scheduled_for))
         .with_for_update()
@@ -131,7 +131,7 @@ def dao_get_future_scheduled_job_by_id_and_service_id(job_id, service_id):
         Job.service_id == service_id,
         Job.id == job_id,
         Job.job_status == JobStatus.SCHEDULED,
-        Job.scheduled_for > datetime.utcnow(),
+        Job.scheduled_for > utc_now(),
     ).one()
 
 
@@ -152,7 +152,7 @@ def dao_get_jobs_older_than_data_retention(notification_types):
         ServiceDataRetention.notification_type.in_(notification_types)
     ).all()
     jobs = []
-    today = datetime.utcnow().date()
+    today = utc_now().date()
     for f in flexible_data_retention:
         end_date = today - timedelta(days=f.days_of_retention)
 
@@ -193,8 +193,8 @@ def dao_get_jobs_older_than_data_retention(notification_types):
 def find_jobs_with_missing_rows():
     # Jobs can be a maximum of 100,000 rows. It typically takes 10 minutes to create all those notifications.
     # Using 20 minutes as a condition seems reasonable.
-    ten_minutes_ago = datetime.utcnow() - timedelta(minutes=20)
-    yesterday = datetime.utcnow() - timedelta(days=1)
+    ten_minutes_ago = utc_now() - timedelta(minutes=20)
+    yesterday = utc_now() - timedelta(days=1)
     jobs_with_rows_missing = (
         db.session.query(Job)
         .filter(

app/dao/notifications_dao.py

@@ -1,4 +1,4 @@
-from datetime import datetime, timedelta
+from datetime import timedelta
 
 from flask import current_app
 from sqlalchemy import asc, desc, or_, select, text, union
@@ -16,6 +16,7 @@ from app.utils import (
     escape_special_characters,
     get_midnight_in_utc,
     midnight_n_days_ago,
+    utc_now,
 )
 from notifications_utils.international_billing_rates import INTERNATIONAL_BILLING_RATES
 from notifications_utils.recipients import (
@@ -95,7 +96,7 @@ def _update_notification_status(
         current_status=notification.status, status=status
     )
     notification.status = status
-    notification.sent_at = datetime.utcnow()
+    notification.sent_at = utc_now()
     if provider_response:
         notification.provider_response = provider_response
     if carrier:
@@ -179,7 +180,7 @@ def update_notification_status_by_reference(reference, status):
 
 @autocommit
 def dao_update_notification(notification):
-    notification.updated_at = datetime.utcnow()
+    notification.updated_at = utc_now()
     # notify-api-742 remove phone numbers from db
     notification.to = "1"
     notification.normalised_to = "1"
@@ -327,7 +328,7 @@ def sanitize_successful_notification_by_id(notification_id, carrier, provider_re
         "notification_id": notification_id,
         "carrier": carrier,
         "response": provider_response,
-        "sent_at": datetime.utcnow(),
+        "sent_at": utc_now(),
     }
 
     db.session.execute(text(update_query), input_params)
@@ -437,7 +438,7 @@ def dao_timeout_notifications(cutoff_time, limit=100000):
     Set email and SMS notifications (only) to "temporary-failure" status
     if they're still sending from before the specified cutoff_time.
     """
-    updated_at = datetime.utcnow()
+    updated_at = utc_now()
     current_statuses = [NotificationStatus.SENDING, NotificationStatus.PENDING]
     new_status = NotificationStatus.TEMPORARY_FAILURE
 
@@ -599,9 +600,7 @@ def dao_get_last_notification_added_for_job_id(job_id):
 
 
 def notifications_not_yet_sent(should_be_sending_after_seconds, notification_type):
-    older_than_date = datetime.utcnow() - timedelta(
-        seconds=should_be_sending_after_seconds
-    )
+    older_than_date = utc_now() - timedelta(seconds=should_be_sending_after_seconds)
 
     notifications = Notification.query.filter(
         Notification.created_at <= older_than_date,
@@ -622,8 +621,7 @@ def _duplicate_update_warning(notification, status):
             id=notification.id,
             old_status=notification.status,
             new_status=status,
-            time_diff=datetime.utcnow()
-            - (notification.updated_at or notification.created_at),
+            time_diff=utc_now() - (notification.updated_at or notification.created_at),
             type=notification.notification_type,
             sent_by=notification.sent_by,
             service_id=notification.service_id,

app/dao/provider_details_dao.py

@@ -7,6 +7,7 @@ from app import db
 from app.dao.dao_utils import autocommit
 from app.enums import NotificationType
 from app.models import FactBilling, ProviderDetails, ProviderDetailsHistory, User
+from app.utils import utc_now
 
 
 def get_provider_details_by_id(provider_details_id):
@@ -66,7 +67,7 @@ def _get_sms_providers_for_update(time_threshold):
 
     # if something updated recently, don't update again. If the updated_at is null, treat it as min time
     if any(
-        (provider.updated_at or datetime.min) > datetime.utcnow() - time_threshold
+        (provider.updated_at or datetime.min) > utc_now() - time_threshold
         for provider in q
     ):
         current_app.logger.info(
@@ -102,7 +103,7 @@ def _update_provider_details_without_commit(provider_details):
     Doesn't commit, for when you need to control the database transaction manually
     """
     provider_details.version += 1
-    provider_details.updated_at = datetime.utcnow()
+    provider_details.updated_at = utc_now()
     history = ProviderDetailsHistory.from_original(provider_details)
     db.session.add(provider_details)
     db.session.add(history)
@@ -111,7 +112,7 @@ def _update_provider_details_without_commit(provider_details):
 def dao_get_provider_stats():
     # this query does not include the current day since the task to populate ft_billing runs overnight
 
-    current_datetime = datetime.utcnow()
+    current_datetime = utc_now()
     first_day_of_the_month = current_datetime.date().replace(day=1)
 
     subquery = (

app/dao/service_callback_api_dao.py

@@ -1,16 +1,15 @@
-from datetime import datetime
-
 from app import create_uuid, db
 from app.dao.dao_utils import autocommit, version_class
 from app.enums import CallbackType
 from app.models import ServiceCallbackApi
+from app.utils import utc_now
 
 
 @autocommit
 @version_class(ServiceCallbackApi)
 def save_service_callback_api(service_callback_api):
     service_callback_api.id = create_uuid()
-    service_callback_api.created_at = datetime.utcnow()
+    service_callback_api.created_at = utc_now()
     db.session.add(service_callback_api)
 
 
@@ -24,7 +23,7 @@ def reset_service_callback_api(
     if bearer_token:
         service_callback_api.bearer_token = bearer_token
     service_callback_api.updated_by_id = updated_by_id
-    service_callback_api.updated_at = datetime.utcnow()
+    service_callback_api.updated_at = utc_now()
 
     db.session.add(service_callback_api)
 

app/dao/service_data_retention_dao.py

@@ -1,8 +1,7 @@
-from datetime import datetime
-
 from app import db
 from app.dao.dao_utils import autocommit
 from app.models import ServiceDataRetention
+from app.utils import utc_now
 
 
 def fetch_service_data_retention_by_id(service_id, data_retention_id):
@@ -50,7 +49,7 @@ def update_service_data_retention(
     updated_count = ServiceDataRetention.query.filter(
         ServiceDataRetention.id == service_data_retention_id,
         ServiceDataRetention.service_id == service_id,
-    ).update({"days_of_retention": days_of_retention, "updated_at": datetime.utcnow()})
+    ).update({"days_of_retention": days_of_retention, "updated_at": utc_now()})
     return updated_count
 
 

app/dao/service_inbound_api_dao.py

@@ -1,15 +1,14 @@
-from datetime import datetime
-
 from app import create_uuid, db
 from app.dao.dao_utils import autocommit, version_class
 from app.models import ServiceInboundApi
+from app.utils import utc_now
 
 
 @autocommit
 @version_class(ServiceInboundApi)
 def save_service_inbound_api(service_inbound_api):
     service_inbound_api.id = create_uuid()
-    service_inbound_api.created_at = datetime.utcnow()
+    service_inbound_api.created_at = utc_now()
     db.session.add(service_inbound_api)
 
 
@@ -23,7 +22,7 @@ def reset_service_inbound_api(
     if bearer_token:
         service_inbound_api.bearer_token = bearer_token
     service_inbound_api.updated_by_id = updated_by_id
-    service_inbound_api.updated_at = datetime.utcnow()
+    service_inbound_api.updated_at = utc_now()
 
     db.session.add(service_inbound_api)
 

app/dao/services_dao.py

@@ -1,5 +1,5 @@
 import uuid
-from datetime import datetime, timedelta
+from datetime import timedelta
 
 from flask import current_app
 from sqlalchemy import Float, cast, select
@@ -44,6 +44,7 @@ from app.utils import (
     escape_special_characters,
     get_archived_db_column_value,
     get_midnight_in_utc,
+    utc_now,
 )
 
 
@@ -252,7 +253,7 @@ def dao_archive_service(service_id):
 
     for api_key in service.api_keys:
         if not api_key.expiry_date:
-            api_key.expiry_date = datetime.utcnow()
+            api_key.expiry_date = utc_now()
 
 
 def dao_fetch_service_by_id_and_user(service_id, user_id):
@@ -404,7 +405,7 @@ def delete_service_and_all_associated_db_objects(service):
 
 
 def dao_fetch_todays_stats_for_service(service_id):
-    today = datetime.utcnow().date()
+    today = utc_now().date()
     start_date = get_midnight_in_utc(today)
     return (
         db.session.query(
@@ -428,7 +429,7 @@ def dao_fetch_todays_stats_for_service(service_id):
 def dao_fetch_todays_stats_for_all_services(
     include_from_test_key=True, only_active=True
 ):
-    today = datetime.utcnow().date()
+    today = utc_now().date()
     start_date = get_midnight_in_utc(today)
     end_date = get_midnight_in_utc(today + timedelta(days=1))
 
@@ -491,7 +492,7 @@ def dao_suspend_service(service_id):
 
     for api_key in service.api_keys:
         if not api_key.expiry_date:
-            api_key.expiry_date = datetime.utcnow()
+            api_key.expiry_date = utc_now()
 
     service.active = False
 

app/dao/templates_dao.py

@@ -1,11 +1,11 @@
 import uuid
-from datetime import datetime
 
 from sqlalchemy import asc, desc
 
 from app import db
 from app.dao.dao_utils import VersionOptions, autocommit, version_class
 from app.models import Template, TemplateHistory, TemplateRedacted
+from app.utils import utc_now
 
 
 @autocommit
@@ -39,7 +39,7 @@ def dao_update_template(template):
 @autocommit
 def dao_redact_template(template, user_id):
     template.template_redacted.redact_personalisation = True
-    template.template_redacted.updated_at = datetime.utcnow()
+    template.template_redacted.updated_at = utc_now()
     template.template_redacted.updated_by_id = user_id
     db.session.add(template.template_redacted)
 

app/dao/uploads_dao.py

@@ -1,4 +1,3 @@
-from datetime import datetime
 from os import getenv
 
 from flask import current_app
@@ -7,7 +6,7 @@ from sqlalchemy import String, and_, desc, func, literal, text
 from app import db
 from app.enums import JobStatus, NotificationStatus, NotificationType
 from app.models import Job, Notification, ServiceDataRetention, Template
-from app.utils import midnight_n_days_ago
+from app.utils import midnight_n_days_ago, utc_now
 
 
 def _get_printing_day(created_at):
@@ -40,7 +39,7 @@ def _naive_gmt_to_utc(column):
 def dao_get_uploads_by_service_id(service_id, limit_days=None, page=1, page_size=50):
     # Hardcoded filter to exclude cancelled or scheduled jobs
     # for the moment, but we may want to change this method take 'statuses' as a argument in the future
-    today = datetime.utcnow().date()
+    today = utc_now().date()
     jobs_query_filter = [
         Job.service_id == service_id,
         Job.original_file_name != current_app.config["TEST_MESSAGE_FILENAME"],

app/dao/users_dao.py

@@ -1,5 +1,5 @@
 import uuid
-from datetime import datetime, timedelta
+from datetime import timedelta
 from secrets import randbelow
 
 import sqlalchemy
@@ -14,7 +14,7 @@ from app.dao.service_user_dao import dao_get_service_users_by_user_id
 from app.enums import AuthType, PermissionType
 from app.errors import InvalidRequest
 from app.models import Organization, Service, User, VerifyCode
-from app.utils import escape_special_characters, get_archived_db_column_value
+from app.utils import escape_special_characters, get_archived_db_column_value, utc_now
 
 
 def _remove_values_for_keys_if_present(dict, keys):
@@ -76,9 +76,9 @@ def save_model_user(
 ):
     if password:
         user.password = password
-        user.password_changed_at = datetime.utcnow()
+        user.password_changed_at = utc_now()
     if validated_email_access:
-        user.email_access_validated_at = datetime.utcnow()
+        user.email_access_validated_at = utc_now()
     if update_dict:
         _remove_values_for_keys_if_present(update_dict, ["id", "password_changed_at"])
         db.session.query(User).filter_by(id=user.id).update(update_dict or {})
@@ -90,7 +90,7 @@ def save_model_user(
 def create_user_code(user, code, code_type):
     verify_code = VerifyCode(
         code_type=code_type,
-        expiry_datetime=datetime.utcnow() + timedelta(minutes=30),
+        expiry_datetime=utc_now() + timedelta(minutes=30),
         user=user,
     )
     verify_code.code = code
@@ -111,7 +111,7 @@ def get_user_code(user, code, code_type):
 def delete_codes_older_created_more_than_a_day_ago():
     deleted = (
         db.session.query(VerifyCode)
-        .filter(VerifyCode.created_at < datetime.utcnow() - timedelta(hours=24))
+        .filter(VerifyCode.created_at < utc_now() - timedelta(hours=24))
         .delete()
     )
     db.session.commit()
@@ -138,7 +138,7 @@ def delete_user_verify_codes(user):
 def count_user_verify_codes(user):
     query = VerifyCode.query.filter(
         VerifyCode.user == user,
-        VerifyCode.expiry_datetime > datetime.utcnow(),
+        VerifyCode.expiry_datetime > utc_now(),
         VerifyCode.code_used.is_(False),
     )
     return query.count()
@@ -179,7 +179,7 @@ def reset_failed_login_count(user):
 def update_user_password(user, password):
     # reset failed login count - they've just reset their password so should be fine
     user.password = password
-    user.password_changed_at = datetime.utcnow()
+    user.password_changed_at = utc_now()
     db.session.add(user)
     db.session.commit()