Add more debug

wrap calls to get personalization and phone so dashboard doesn't explode

app/aws/s3.py

@@ -1,5 +1,7 @@
 import re
+import csv
 
+from app.utils import hilite
 import botocore
 from boto3 import Session
 from expiringdict import ExpiringDict
@@ -124,17 +126,28 @@ def extract_phones(job):
 
 
 def extract_personalisation(job):
+    print(hilite(f"job type: {type(job)}"))
+    print(hilite(f"Job? {job}"))
+
     job = job.split("\r\n")
+    print(hilite(f"job after first split: {job}"))
     first_row = job[0]
+    print(hilite(f"first_row: {first_row}"))
     job.pop(0)
     first_row = first_row.split(",")
+    print(hilite(f"first_row again: {first_row}"))
     personalisation = {}
     job_row = 0
-    for row in job:
-        row = row.split(",")
+    row_csv_module = csv.reader(job)
+
+    for row in row_csv_module:
+        print(hilite(f"row: {row}"))
         temp = dict(zip(first_row, row))
+        print(hilite(f"temp: {temp}"))
         personalisation[job_row] = temp
         job_row = job_row + 1
+        print(hilite(f"job_row: {job_row}"))
+    print(hilite(f"personalisation: {personalisation}"))
     return personalisation
 
 
@@ -190,6 +203,7 @@ def get_personalisation_from_s3(service_id, job_id, job_row_number):
     job = JOBS.get(job_id)
     if job is None:
         job = get_job_from_s3(service_id, job_id)
+        print(hilite(f"job at the beginning: {job}"))
         JOBS[job_id] = job
         incr_jobs_cache_misses()
     else:

app/celery/provider_tasks.py

@@ -18,7 +18,7 @@ from app.dao.notifications_dao import (
 from app.delivery import send_to_providers
 from app.enums import NotificationStatus
 from app.exceptions import NotificationTechnicalFailureException
-from app.utils import utc_now
+from app.utils import hilite, utc_now
 
 # This is the amount of time to wait after sending an sms message before we check the aws logs and look for delivery
 # receipts
@@ -100,7 +100,7 @@ def deliver_sms(self, notification_id):
         notification = notifications_dao.get_notification_by_id(notification_id)
         ansi_green = "\033[32m"
         ansi_reset = "\033[0m"
-
+        print(hilite(f"notification inside of deliver_sms: {notification}"))
         if not notification:
             raise NoResultFound()
         if (

app/celery/tasks.py

@@ -170,6 +170,7 @@ def __total_sending_limits_for_job_exceeded(service, job, job_id):
 def save_sms(self, service_id, notification_id, encrypted_notification, sender_id=None):
     """Persist notification to db and place notification in queue to send to sns."""
     notification = encryption.decrypt(encrypted_notification)
+    print(hilite(f"notification at the top of save_sms: {notification}"))
     # SerialisedService and SerialisedTemplate classes are
     # used here to grab the same service and template from the cache
     # to improve performance.

app/clients/sms/aws_sns.py

@@ -2,6 +2,7 @@ import os
 import re
 from time import monotonic
 
+from app.utils import hilite
 import botocore
 import phonenumbers
 from boto3 import client
@@ -54,7 +55,7 @@ class AwsSnsClient(SmsClient):
             to = phonenumbers.format_number(
                 match.number, phonenumbers.PhoneNumberFormat.E164
             )
-
+            print(hilite(f"to in send_sms: {to}"))
             # See documentation
             # https://docs.aws.amazon.com/sns/latest/dg/sms_publish-to-phone.html#sms_publish_sdk
             attributes = {

app/commands.py

@@ -593,10 +593,10 @@ def process_row_from_job(job_id, job_row_number):
 @click.option("-f", "--csv_filename", required=True, help="csv file name")
 def download_csv_file_by_name(csv_filename):
 
-    bucket_name = (current_app.config["CSV_UPLOAD_BUCKET"]["bucket"],)
-    access_key = (current_app.config["CSV_UPLOAD_BUCKET"]["access_key_id"],)
-    secret = (current_app.config["CSV_UPLOAD_BUCKET"]["secret_access_key"],)
-    region = (current_app.config["CSV_UPLOAD_BUCKET"]["region"],)
+    bucket_name = current_app.config["CSV_UPLOAD_BUCKET"]["bucket"]
+    access_key = current_app.config["CSV_UPLOAD_BUCKET"]["access_key_id"]
+    secret = current_app.config["CSV_UPLOAD_BUCKET"]["secret_access_key"]
+    region = current_app.config["CSV_UPLOAD_BUCKET"]["region"]
     print(s3.get_s3_file(bucket_name, csv_filename, access_key, secret, region))
 
 
@@ -865,14 +865,12 @@ def promote_user_to_platform_admin(user_email_address):
 
 @notify_command(name="purge-csv-bucket")
 def purge_csv_bucket():
-    bucket_name = (current_app.config["CSV_UPLOAD_BUCKET"]["bucket"],)
-    access_key = (current_app.config["CSV_UPLOAD_BUCKET"]["access_key_id"],)
-    secret = (current_app.config["CSV_UPLOAD_BUCKET"]["secret_access_key"],)
-    region = (current_app.config["CSV_UPLOAD_BUCKET"]["region"],)
+    bucket_name = current_app.config["CSV_UPLOAD_BUCKET"]["bucket"]
+    access_key = current_app.config["CSV_UPLOAD_BUCKET"]["access_key_id"]
+    secret = current_app.config["CSV_UPLOAD_BUCKET"]["secret_access_key"]
+    region = current_app.config["CSV_UPLOAD_BUCKET"]["region"]
 
-    print("ABOUT TO RUN PURGE CSV BUCKET")
     s3.purge_bucket(bucket_name, access_key, secret, region)
-    print("RAN PURGE CSV BUCKET")
 
 
 """

app/dao/date_util.py

@@ -1,3 +1,4 @@
+import calendar
 from datetime import date, datetime, time, timedelta
 
 from app.utils import utc_now
@@ -66,3 +67,29 @@ def get_calendar_year_for_datetime(start_date):
         return year - 1
     else:
         return year
+
+
+def get_number_of_days_for_month(year, month):
+    return calendar.monthrange(year, month)[1]
+
+
+def generate_date_range(start_date, end_date=None, days=0):
+    if end_date:
+        current_date = start_date
+        while current_date <= end_date:
+            try:
+                yield current_date.date()
+            except ValueError:
+                pass
+            current_date += timedelta(days=1)
+    elif days > 0:
+        end_date = start_date + timedelta(days=days)
+        current_date = start_date
+        while current_date < end_date:
+            try:
+                yield current_date.date()
+            except ValueError:
+                pass
+            current_date += timedelta(days=1)
+    else:
+        return "An end_date or number of days must be specified"

app/dao/fact_notification_status_dao.py

@@ -84,21 +84,21 @@ def update_fact_notification_status(process_day, notification_type, service_id):
 def fetch_notification_status_for_service_by_month(start_date, end_date, service_id):
     return (
         db.session.query(
-            func.date_trunc("month", FactNotificationStatus.local_date).label("month"),
-            FactNotificationStatus.notification_type,
-            FactNotificationStatus.notification_status,
-            func.sum(FactNotificationStatus.notification_count).label("count"),
+            func.date_trunc("month", NotificationAllTimeView.created_at).label("month"),
+            NotificationAllTimeView.notification_type,
+            NotificationAllTimeView.status.label("notification_status"),
+            func.count(NotificationAllTimeView.id).label("count"),
         )
         .filter(
-            FactNotificationStatus.service_id == service_id,
-            FactNotificationStatus.local_date >= start_date,
-            FactNotificationStatus.local_date < end_date,
-            FactNotificationStatus.key_type != KeyType.TEST,
+            NotificationAllTimeView.service_id == service_id,
+            NotificationAllTimeView.created_at >= start_date,
+            NotificationAllTimeView.created_at < end_date,
+            NotificationAllTimeView.key_type != KeyType.TEST,
         )
         .group_by(
-            func.date_trunc("month", FactNotificationStatus.local_date).label("month"),
-            FactNotificationStatus.notification_type,
-            FactNotificationStatus.notification_status,
+            func.date_trunc("month", NotificationAllTimeView.created_at).label("month"),
+            NotificationAllTimeView.notification_type,
+            NotificationAllTimeView.status,
         )
         .all()
     )

app/dao/services_dao.py

@@ -8,7 +8,7 @@ from sqlalchemy.sql.expression import and_, asc, case, func
 
 from app import db
 from app.dao.dao_utils import VersionOptions, autocommit, version_class
-from app.dao.date_util import get_current_calendar_year
+from app.dao.date_util import generate_date_range, get_current_calendar_year
 from app.dao.organization_dao import dao_get_organization_by_email_address
 from app.dao.service_sms_sender_dao import insert_service_sms_sender
 from app.dao.service_user_dao import dao_get_service_user
@@ -27,6 +27,7 @@ from app.models import (
     InvitedUser,
     Job,
     Notification,
+    NotificationAllTimeView,
     NotificationHistory,
     Organization,
     Permission,
@@ -40,6 +41,7 @@ from app.models import (
     User,
     VerifyCode,
 )
+from app.service import statistics
 from app.utils import (
     escape_special_characters,
     get_archived_db_column_value,
@@ -426,6 +428,61 @@ def dao_fetch_todays_stats_for_service(service_id):
     )
 
 
+def dao_fetch_stats_for_service_from_days(service_id, start_date, end_date):
+    start_date = get_midnight_in_utc(start_date)
+    end_date = get_midnight_in_utc(end_date + timedelta(days=1))
+
+    return (
+        db.session.query(
+            NotificationAllTimeView.notification_type,
+            NotificationAllTimeView.status,
+            func.date_trunc("day", NotificationAllTimeView.created_at).label("day"),
+            func.count(NotificationAllTimeView.id).label("count"),
+        )
+        .filter(
+            NotificationAllTimeView.service_id == service_id,
+            NotificationAllTimeView.key_type != KeyType.TEST,
+            NotificationAllTimeView.created_at >= start_date,
+            NotificationAllTimeView.created_at < end_date,
+        )
+        .group_by(
+            NotificationAllTimeView.notification_type,
+            NotificationAllTimeView.status,
+            func.date_trunc("day", NotificationAllTimeView.created_at),
+        )
+        .all()
+    )
+
+
+def dao_fetch_stats_for_service_from_days_for_user(
+    service_id, start_date, end_date, user_id
+):
+    start_date = get_midnight_in_utc(start_date)
+    end_date = get_midnight_in_utc(end_date + timedelta(days=1))
+
+    return (
+        db.session.query(
+            NotificationAllTimeView.notification_type,
+            NotificationAllTimeView.status,
+            func.date_trunc("day", NotificationAllTimeView.created_at).label("day"),
+            func.count(NotificationAllTimeView.id).label("count"),
+        )
+        .filter(
+            NotificationAllTimeView.service_id == service_id,
+            NotificationAllTimeView.key_type != KeyType.TEST,
+            NotificationAllTimeView.created_at >= start_date,
+            NotificationAllTimeView.created_at < end_date,
+            NotificationAllTimeView.created_by_id == user_id,
+        )
+        .group_by(
+            NotificationAllTimeView.notification_type,
+            NotificationAllTimeView.status,
+            func.date_trunc("day", NotificationAllTimeView.created_at),
+        )
+        .all()
+    )
+
+
 def dao_fetch_todays_stats_for_all_services(
     include_from_test_key=True, only_active=True
 ):
@@ -607,3 +664,52 @@ def get_live_services_with_organization():
     )
 
     return query.all()
+
+
+def fetch_notification_stats_for_service_by_month_by_user(
+    start_date, end_date, service_id, user_id
+):
+    return (
+        db.session.query(
+            func.date_trunc("month", NotificationAllTimeView.created_at).label("month"),
+            NotificationAllTimeView.notification_type,
+            (NotificationAllTimeView.status).label("notification_status"),
+            func.count(NotificationAllTimeView.id).label("count"),
+        )
+        .filter(
+            NotificationAllTimeView.service_id == service_id,
+            NotificationAllTimeView.created_at >= start_date,
+            NotificationAllTimeView.created_at < end_date,
+            NotificationAllTimeView.key_type != KeyType.TEST,
+            NotificationAllTimeView.created_by_id == user_id,
+        )
+        .group_by(
+            func.date_trunc("month", NotificationAllTimeView.created_at).label("month"),
+            NotificationAllTimeView.notification_type,
+            NotificationAllTimeView.status,
+        )
+        .all()
+    )
+
+
+def get_specific_days_stats(results, start_date, days=None, end_date=None):
+    if days is not None and end_date is not None:
+        raise ValueError("Only set days OR set end_date, not both.")
+    elif days is not None:
+        gen_range = generate_date_range(start_date, days=days)
+    elif end_date is not None:
+        gen_range = generate_date_range(start_date, end_date)
+    else:
+        raise ValueError("Either days or end_date must be set.")
+
+    grouped_results = {date: [] for date in gen_range} | {
+        day.date(): [notification_type, status, day, count]
+        for notification_type, status, day, count in results
+    }
+
+    stats = {
+        day.strftime("%Y-%m-%d"): statistics.format_statistics(rows)
+        for day, rows in grouped_results.items()
+    }
+
+    return stats

app/dao/users_dao.py

@@ -4,7 +4,7 @@ from secrets import randbelow
 
 import sqlalchemy
 from flask import current_app
-from sqlalchemy import func
+from sqlalchemy import func, text
 from sqlalchemy.orm import joinedload
 
 from app import db
@@ -244,3 +244,15 @@ def user_can_be_archived(user):
             return False
 
     return True
+
+
+def dao_report_users():
+    sql = """
+    select users.name, users.email_address, users.mobile_number, services.name as service_name
+    from users
+    inner join user_to_service on users.id=user_to_service.user_id
+    inner join services on services.id=user_to_service.service_id
+    where services.name not like '_archived%'
+    order by services.name asc, users.name asc
+    """
+    return db.session.execute(text(sql))

app/delivery/send_to_providers.py

@@ -86,6 +86,7 @@ def send_sms_to_provider(notification):
                         notification.job_id,
                         notification.job_row_number,
                     )
+                    print(hilite(f"recipient: {recipient}"))
                 except Exception:
                     # It is our 2facode, maybe
                     key = f"2facode-{notification.id}".replace(" ", "")
@@ -108,6 +109,7 @@ def send_sms_to_provider(notification):
                     "sender": notification.reply_to_text,
                     "international": notification.international,
                 }
+                print(hilite(f"send_sms_kwargs: {send_sms_kwargs}"))
                 db.session.close()  # no commit needed as no changes to objects have been made above
 
                 message_id = provider.send_sms(**send_sms_kwargs)

app/job/rest.py

@@ -33,7 +33,7 @@ from app.schemas import (
     notifications_filter_schema,
     unarchived_template_schema,
 )
-from app.utils import midnight_n_days_ago, pagination_links
+from app.utils import hilite, midnight_n_days_ago, pagination_links
 
 job_blueprint = Blueprint("job", __name__, url_prefix="/service/<uuid:service_id>/job")
 
@@ -172,6 +172,7 @@ def create_job(service_id):
         raise InvalidRequest("Create job is not allowed: service is inactive ", 403)
 
     data = request.get_json()
+    print(hilite(f"data at the top of create_job: {data}"))
     original_file_name = data.get("original_file_name")
     data.update({"service": service_id})
     try:

app/service/rest.py

@@ -1,6 +1,7 @@
 import itertools
-from datetime import datetime
+from datetime import datetime, timedelta
 
+from botocore.exceptions import ClientError
 from flask import Blueprint, current_app, jsonify, request
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm.exc import NoResultFound
@@ -17,7 +18,7 @@ from app.dao.api_key_dao import (
     save_model_api_key,
 )
 from app.dao.dao_utils import dao_rollback, transaction
-from app.dao.date_util import get_calendar_year
+from app.dao.date_util import get_calendar_year, get_month_start_and_end_date_in_utc
 from app.dao.fact_notification_status_dao import (
     fetch_monthly_template_usage_for_service,
     fetch_notification_status_for_service_by_month,
@@ -63,13 +64,17 @@ from app.dao.services_dao import (
     dao_fetch_all_services_by_user,
     dao_fetch_live_services_data,
     dao_fetch_service_by_id,
+    dao_fetch_stats_for_service_from_days,
+    dao_fetch_stats_for_service_from_days_for_user,
     dao_fetch_todays_stats_for_all_services,
     dao_fetch_todays_stats_for_service,
     dao_remove_user_from_service,
     dao_resume_service,
     dao_suspend_service,
     dao_update_service,
+    fetch_notification_stats_for_service_by_month_by_user,
     get_services_by_partial_name,
+    get_specific_days_stats,
 )
 from app.dao.templates_dao import dao_get_template_by_id
 from app.dao.users_dao import get_user_by_id
@@ -210,6 +215,58 @@ def get_service_notification_statistics(service_id):
     )
 
 
+@service_blueprint.route("/<uuid:service_id>/statistics/<string:start>/<int:days>")
+def get_service_notification_statistics_by_day(service_id, start, days):
+    return jsonify(
+        data=get_service_statistics_for_specific_days(service_id, start, int(days))
+    )
+
+
+def get_service_statistics_for_specific_days(service_id, start, days=1):
+    # start and end dates needs to be reversed because
+    # the end date is today and the start is x days in the past
+    # a day needs to be substracted to allow for today
+    end_date = datetime.strptime(start, "%Y-%m-%d")
+    start_date = end_date - timedelta(days=days - 1)
+
+    results = dao_fetch_stats_for_service_from_days(service_id, start_date, end_date)
+
+    stats = get_specific_days_stats(results, start_date, days=days)
+
+    return stats
+
+
+@service_blueprint.route(
+    "/<uuid:service_id>/statistics/user/<uuid:user_id>/<string:start>/<int:days>"
+)
+def get_service_notification_statistics_by_day_by_user(
+    service_id, user_id, start, days
+):
+    return jsonify(
+        data=get_service_statistics_for_specific_days_by_user(
+            service_id, user_id, start, int(days)
+        )
+    )
+
+
+def get_service_statistics_for_specific_days_by_user(
+    service_id, user_id, start, days=1
+):
+    # start and end dates needs to be reversed because
+    # the end date is today and the start is x days in the past
+    # a day needs to be substracted to allow for today
+    end_date = datetime.strptime(start, "%Y-%m-%d")
+    start_date = end_date - timedelta(days=days - 1)
+
+    results = dao_fetch_stats_for_service_from_days_for_user(
+        service_id, start_date, end_date, user_id
+    )
+
+    stats = get_specific_days_stats(results, start_date, days=days)
+
+    return stats
+
+
 @service_blueprint.route("", methods=["POST"])
 def create_service():
     data = request.get_json()
@@ -444,18 +501,44 @@ def get_all_notifications_for_service(service_id):
 
     for notification in pagination.items:
         if notification.job_id is not None:
-            notification.personalisation = get_personalisation_from_s3(
-                notification.service_id,
-                notification.job_id,
-                notification.job_row_number,
-            )
-            recipient = get_phone_number_from_s3(
-                notification.service_id,
-                notification.job_id,
-                notification.job_row_number,
-            )
-            notification.to = recipient
-            notification.normalised_to = recipient
+            try:
+                notification.personalisation = get_personalisation_from_s3(
+                    notification.service_id,
+                    notification.job_id,
+                    notification.job_row_number,
+                )
+            except ClientError as ex:
+                if ex.response["Error"]["Code"] == "NoSuchKey":
+                    s = notification.service_id
+                    j = notification.job_id
+                    current_app.logger.warning(
+                        f"No personalisation found for s3 file location service: service-{s}-notify/{j}.csv"
+                    )
+                    notification.personalisation = ""
+                else:
+                    raise ex
+
+            try:
+                recipient = get_phone_number_from_s3(
+                    notification.service_id,
+                    notification.job_id,
+                    notification.job_row_number,
+                )
+
+                notification.to = recipient
+                notification.normalised_to = recipient
+            except ClientError as ex:
+                if ex.response["Error"]["Code"] == "NoSuchKey":
+                    s = notification.service_id
+                    j = notification.job_id
+                    current_app.logger.warning(
+                        f"No phone number found for s3 file location service: service-{s}-notify/{j}.csv"
+                    )
+                    notification.to = ""
+                    notification.normalised_to = ""
+                else:
+                    raise ex
+
         else:
             notification.to = "1"
             notification.normalised_to = "1"
@@ -592,6 +675,7 @@ def get_monthly_notification_stats(service_id):
     stats = fetch_notification_status_for_service_by_month(
         start_date, end_date, service_id
     )
+
     statistics.add_monthly_notification_status_stats(data, stats)
 
     now = utc_now()
@@ -604,6 +688,87 @@ def get_monthly_notification_stats(service_id):
     return jsonify(data=data)
 
 
+@service_blueprint.route(
+    "/<uuid:service_id>/notifications/<uuid:user_id>/monthly", methods=["GET"]
+)
+def get_monthly_notification_stats_by_user(service_id, user_id):
+    # check service_id validity
+    dao_fetch_service_by_id(service_id)
+    # user = get_user_by_id(user_id=user_id)
+
+    try:
+        year = int(request.args.get("year", "NaN"))
+    except ValueError:
+        raise InvalidRequest("Year must be a number", status_code=400)
+
+    start_date, end_date = get_calendar_year(year)
+
+    data = statistics.create_empty_monthly_notification_status_stats_dict(year)
+
+    stats = fetch_notification_stats_for_service_by_month_by_user(
+        start_date, end_date, service_id, user_id
+    )
+
+    statistics.add_monthly_notification_status_stats(data, stats)
+
+    now = utc_now()
+    if end_date > now:
+        todays_deltas = fetch_notification_status_for_service_for_day(
+            now, service_id=service_id
+        )
+        statistics.add_monthly_notification_status_stats(data, todays_deltas)
+
+    return jsonify(data=data)
+
+
+@service_blueprint.route(
+    "/<uuid:service_id>/notifications/<uuid:user_id>/month", methods=["GET"]
+)
+def get_single_month_notification_stats_by_user(service_id, user_id):
+    # check service_id validity
+    dao_fetch_service_by_id(service_id)
+
+    try:
+        month = int(request.args.get("month", "NaN"))
+        year = int(request.args.get("year", "NaN"))
+    except ValueError:
+        raise InvalidRequest(
+            "Both a month and year are required as numbers", status_code=400
+        )
+
+    month_year = datetime(year, month, 10, 00, 00, 00)
+    start_date, end_date = get_month_start_and_end_date_in_utc(month_year)
+
+    results = dao_fetch_stats_for_service_from_days_for_user(
+        service_id, start_date, end_date, user_id
+    )
+
+    stats = get_specific_days_stats(results, start_date, end_date=end_date)
+    return jsonify(stats)
+
+
+@service_blueprint.route("/<uuid:service_id>/notifications/month", methods=["GET"])
+def get_single_month_notification_stats_for_service(service_id):
+    # check service_id validity
+    dao_fetch_service_by_id(service_id)
+
+    try:
+        month = int(request.args.get("month", "NaN"))
+        year = int(request.args.get("year", "NaN"))
+    except ValueError:
+        raise InvalidRequest(
+            "Both a month and year are required as numbers", status_code=400
+        )
+
+    month_year = datetime(year, month, 10, 00, 00, 00)
+    start_date, end_date = get_month_start_and_end_date_in_utc(month_year)
+
+    results = dao_fetch_stats_for_service_from_days(service_id, start_date, end_date)
+
+    stats = get_specific_days_stats(results, start_date, end_date=end_date)
+    return jsonify(stats)
+
+
 def get_detailed_service(service_id, today_only=False):
     service = dao_fetch_service_by_id(service_id)
 

app/service/statistics.py

@@ -113,7 +113,6 @@ def create_empty_monthly_notification_status_stats_dict(year):
 def add_monthly_notification_status_stats(data, stats):
     for row in stats:
         month = row.month.strftime("%Y-%m")
-
         data[month][row.notification_type][row.notification_status] += row.count
-
+        data[month][row.notification_type][StatisticsType.REQUESTED] += row.count
     return data

app/user/rest.py

@@ -18,6 +18,7 @@ from app.dao.users_dao import (
     create_secret_code,
     create_user_code,
     dao_archive_user,
+    dao_report_users,
     get_login_gov_user,
     get_user_and_accounts,
     get_user_by_email,
@@ -667,6 +668,12 @@ def update_password(user_id):
     return jsonify(data=user.serialize()), 200
 
 
+@user_blueprint.route("/report-all-users", methods=["GET"])
+def report_all_users():
+    users = dao_report_users()
+    return jsonify(data=users.serialize()), 200
+
+
 @user_blueprint.route("/<uuid:user_id>/organizations-and-services", methods=["GET"])
 def get_organizations_and_services_for_user(user_id):
     user = get_user_and_accounts(user_id)

docs/all.md

@@ -505,7 +505,7 @@ flask command purge_functional_test_data -u <functional tests user name prefix>
 Running on cloud.gov:
 
 ```
-cf run-task notify-api "flask command purge_functional_test_data -u <functional tests user name prefix>"
+cf run-task notify-api --command "flask command purge_functional_test_data -u <functional tests user name prefix>"
 ```
 
 
@@ -1339,9 +1339,24 @@ In the api logs, search by job_id.  Either you will see evidence of the job fail
 
 ## Viewing the csv file
 
-If you need to view the questionable csv file, run the following command:
+If you need to view the questionable csv file on production, run the following command:
 
 
 ```
-cf run-task notify-api "flask command download_csv_file_by_name -f <file location found in admin logs>"
+cf run-task notify-api-production --command "flask command download-csv-file-by-name -f <file location found in admin logs>"
 ```
+
+locally, just do:
+
+```
+poetry run flask command download-csv-file-by-name -f <file location in admin logs>
+```
+
+## Debug steps
+
+1. Either send a message and capture the csv file name, or get a csv file name from a user
+2. Using the log tool at logs.fr.cloud.gov, use filters to limit what you're searching on (cf.app is 'notify-admin-production' for example) and then search with the csv file name in double quotes over the relevant time period (last 5 minutes if you just sent a message, or else whatever time the user sent at)
+3. When you find the log line, you should also find the job_id and the s3 file location.  Save these somewhere.
+4. To get the csv file contents, you can run the command above.  This command currently prints to the notify-api log, so after you run the command,
+you need to search in notify-api-production for the last 5 minutes with the logs sorted by timestamp.  The contents of the csv file unfortunately appear on separate lines so it's very important to sort by time.
+5. If you want to see where the message actually failed, search with cf.app is notify-api-production using the job_id that you saved in step #3.   If you get far enough, you might see one of the log lines has a message_id.  If you see it, you can switch and search on that, which should tell you what happened in AWS (success or failure).

migrations/versions/0025_notify_service_data.py

@@ -15,6 +15,7 @@ from alembic import op
 from sqlalchemy import text
 
 from app.hashing import hashpw
+from app.utils import utc_now
 
 revision = "0025_notify_service_data"
 down_revision = "0024_add_research_mode_defaults"
@@ -32,7 +33,7 @@ def upgrade():
                   """
     conn.execute(
         text(user_insert),
-        {"user_id": user_id, "time_now": datetime.utcnow(), "password": password},
+        {"user_id": user_id, "time_now": utc_now(), "password": password},
     )
     service_history_insert = """INSERT INTO services_history (id, name, created_at, active, message_limit, restricted, research_mode, email_from, created_by_id, reply_to_email_address, version)
                         VALUES (:service_id, 'Notify service', :time_now, True, 1000, False, False, 'testsender@dispostable.com',
@@ -41,7 +42,7 @@ def upgrade():
                      """
     conn.execute(
         text(service_history_insert),
-        {"service_id": service_id, "time_now": datetime.utcnow(), "user_id": user_id},
+        {"service_id": service_id, "time_now": utc_now(), "user_id": user_id},
     )
     service_insert = """INSERT INTO services (id, name, created_at, active, message_limit, restricted, research_mode, email_from, created_by_id, reply_to_email_address, version)
                         VALUES (:service_id, 'Notify service', :time_now, True, 1000, False, False, 'testsender@dispostable.com',
@@ -49,7 +50,7 @@ def upgrade():
                     """
     conn.execute(
         text(service_insert),
-        {"service_id": service_id, "time_now": datetime.utcnow(), "user_id": user_id},
+        {"service_id": service_id, "time_now": utc_now(), "user_id": user_id},
     )
     user_to_service_insert = """INSERT INTO user_to_service (user_id, service_id) VALUES (:user_id, :service_id)"""
     conn.execute(
@@ -74,7 +75,7 @@ def upgrade():
             "template_id": uuid.uuid4(),
             "template_name": "Notify email verification code",
             "template_type": "email",
-            "time_now": datetime.utcnow(),
+            "time_now": utc_now(),
             "content": email_verification_content,
             "service_id": service_id,
             "subject": "Confirm GOV.UK Notify registration",
@@ -87,7 +88,7 @@ def upgrade():
             "template_id": "ece42649-22a8-4d06-b87f-d52d5d3f0a27",
             "template_name": "Notify email verification code",
             "template_type": "email",
-            "time_now": datetime.utcnow(),
+            "time_now": utc_now(),
             "content": email_verification_content,
             "service_id": service_id,
             "subject": "Confirm GOV.UK Notify registration",
@@ -107,7 +108,7 @@ def upgrade():
             "template_id": "4f46df42-f795-4cc4-83bb-65ca312f49cc",
             "template_name": "Notify invitation email",
             "template_type": "email",
-            "time_now": datetime.utcnow(),
+            "time_now": utc_now(),
             "content": invitation_content,
             "service_id": service_id,
             "subject": invitation_subject,
@@ -120,7 +121,7 @@ def upgrade():
             "template_id": "4f46df42-f795-4cc4-83bb-65ca312f49cc",
             "template_name": "Notify invitation email",
             "template_type": "email",
-            "time_now": datetime.utcnow(),
+            "time_now": utc_now(),
             "content": invitation_content,
             "service_id": service_id,
             "subject": invitation_subject,
@@ -135,7 +136,7 @@ def upgrade():
             "template_id": "36fb0730-6259-4da1-8a80-c8de22ad4246",
             "template_name": "Notify SMS verify code",
             "template_type": "sms",
-            "time_now": datetime.utcnow(),
+            "time_now": utc_now(),
             "content": sms_code_content,
             "service_id": service_id,
             "subject": None,
@@ -149,7 +150,7 @@ def upgrade():
             "template_id": "36fb0730-6259-4da1-8a80-c8de22ad4246",
             "template_name": "Notify SMS verify code",
             "template_type": "sms",
-            "time_now": datetime.utcnow(),
+            "time_now": utc_now(),
             "content": sms_code_content,
             "service_id": service_id,
             "subject": None,
@@ -172,7 +173,7 @@ def upgrade():
             "template_id": "474e9242-823b-4f99-813d-ed392e7f1201",
             "template_name": "Notify password reset email",
             "template_type": "email",
-            "time_now": datetime.utcnow(),
+            "time_now": utc_now(),
             "content": password_reset_content,
             "service_id": service_id,
             "subject": "Reset your GOV.UK Notify password",
@@ -186,7 +187,7 @@ def upgrade():
             "template_id": "474e9242-823b-4f99-813d-ed392e7f1201",
             "template_name": "Notify password reset email",
             "template_type": "email",
-            "time_now": datetime.utcnow(),
+            "time_now": utc_now(),
             "content": password_reset_content,
             "service_id": service_id,
             "subject": "Reset your GOV.UK Notify password",

migrations/versions/0028_fix_reg_template_history.py

@@ -11,6 +11,8 @@ from datetime import datetime
 
 from sqlalchemy import text
 
+from app.utils import utc_now
+
 revision = "0028_fix_reg_template_history"
 down_revision = "0026_rename_notify_service"
 
@@ -38,7 +40,7 @@ def upgrade():
         "id": "ece42649-22a8-4d06-b87f-d52d5d3f0a27",
         "name": "Notify email verification code",
         "type": "email",
-        "time_now": datetime.utcnow(),
+        "time_now": utc_now(),
         "content": email_verification_content,
         "service_id": service_id,
         "subject": "Confirm GOV.UK Notify registration",

migrations/versions/0082_add_golive_template.py

@@ -14,6 +14,8 @@ from alembic import op
 from flask import current_app
 from sqlalchemy import text
 
+from app.utils import utc_now
+
 revision = "0082_add_go_live_template"
 down_revision = "0081_noti_status_as_enum"
 
@@ -89,7 +91,7 @@ GOV.UK Notify team
         "template_id": template_id,
         "template_name": template_name,
         "template_type": "email",
-        "time_now": datetime.utcnow(),
+        "time_now": utc_now(),
         "content": template_content,
         "notify_service_id": current_app.config["NOTIFY_SERVICE_ID"],
         "subject": template_subject,

migrations/versions/0117_international_sms_notify.py

@@ -9,6 +9,8 @@ Create Date: 2017-08-29 14:09:41.042061
 # revision identifiers, used by Alembic.
 from sqlalchemy import text
 
+from app.utils import utc_now
+
 revision = "0117_international_sms_notify"
 down_revision = "0115_add_inbound_numbers"
 
@@ -22,7 +24,7 @@ NOTIFY_SERVICE_ID = "d6aa2c68-a2d9-4437-ab19-3ae8eb202553"
 def upgrade():
     input_params = {
         "notify_service_id": NOTIFY_SERVICE_ID,
-        "datetime_now": datetime.utcnow(),
+        "datetime_now": utc_now(),
     }
     conn = op.get_bind()
     conn.execute(

migrations/versions/0134_add_email_2fa_template_.py

@@ -12,6 +12,8 @@ from alembic import op
 from flask import current_app
 from sqlalchemy import text
 
+from app.utils import utc_now
+
 revision = "0134_add_email_2fa_template"
 down_revision = "0133_set_services_sms_prefix"
 
@@ -44,7 +46,7 @@ def upgrade():
         "template_id": template_id,
         "template_name": template_name,
         "template_type": "email",
-        "time_now": datetime.utcnow(),
+        "time_now": utc_now(),
         "content": template_content,
         "notify_service_id": current_app.config["NOTIFY_SERVICE_ID"],
         "subject": template_subject,

migrations/versions/0139_migrate_sms_allowance_data.py

@@ -13,6 +13,7 @@ from alembic import op
 from sqlalchemy import text
 
 from app.dao.date_util import get_current_calendar_year_start_year
+from app.utils import utc_now
 
 revision = "0139_migrate_sms_allowance_data"
 down_revision = "0138_sms_sender_nullable"
@@ -34,7 +35,7 @@ def upgrade():
     input_params = {
         "current_year": current_year,
         "default_limit": default_limit,
-        "time_now": datetime.utcnow(),
+        "time_now": utc_now(),
     }
     insert_row_if_not_exist = """
         INSERT INTO annual_billing

migrations/versions/0171_add_org_invite_template.py

@@ -12,6 +12,8 @@ from alembic import op
 from flask import current_app
 from sqlalchemy import text
 
+from app.utils import utc_now
+
 revision = "0171_add_org_invite_template"
 down_revision = "0170_hidden_non_nullable"
 
@@ -53,7 +55,7 @@ def upgrade():
         "template_id": template_id,
         "template_name": template_name,
         "template_type": "email",
-        "time_now": datetime.utcnow(),
+        "time_now": utc_now(),
         "content": template_content,
         "notify_service_id": current_app.config["NOTIFY_SERVICE_ID"],
         "subject": template_subject,

migrations/versions/0265_add_confirm_edit_templates.py

@@ -12,6 +12,8 @@ from alembic import op
 from flask import current_app
 from sqlalchemy import text
 
+from app.utils import utc_now
+
 revision = "0265_add_confirm_edit_templates"
 down_revision = "0264_add_folder_permissions_perm"
 
@@ -57,7 +59,7 @@ def upgrade():
         "template_id": email_template_id,
         "template_name": email_template_name,
         "template_type": "email",
-        "time_now": datetime.utcnow(),
+        "time_now": utc_now(),
         "content": email_template_content,
         "notify_service_id": current_app.config["NOTIFY_SERVICE_ID"],
         "subject": email_template_subject,
@@ -78,7 +80,7 @@ def upgrade():
         "template_id": mobile_template_id,
         "template_name": mobile_template_name,
         "template_type": "sms",
-        "time_now": datetime.utcnow(),
+        "time_now": utc_now(),
         "content": mobile_template_content,
         "notify_service_id": current_app.config["NOTIFY_SERVICE_ID"],
         "subject": None,

migrations/versions/0294_add_verify_reply_to_.py

@@ -12,6 +12,8 @@ from alembic import op
 from flask import current_app
 from sqlalchemy import text
 
+from app.utils import utc_now
+
 revision = "0294_add_verify_reply_to"
 down_revision = "0293_drop_complaint_fk"
 
@@ -58,7 +60,7 @@ def upgrade():
         "template_id": email_template_id,
         "template_name": email_template_name,
         "template_type": "email",
-        "time_now": datetime.utcnow(),
+        "time_now": utc_now(),
         "content": email_template_content,
         "notify_service_id": current_app.config["NOTIFY_SERVICE_ID"],
         "subject": email_template_subject,

migrations/versions/0330_broadcast_invite_email.py

@@ -12,6 +12,8 @@ from datetime import datetime
 from alembic import op
 from sqlalchemy import text
 
+from app.utils import utc_now
+
 revision = "0330_broadcast_invite_email"
 down_revision = "0329_purge_broadcast_data"
 
@@ -60,7 +62,7 @@ def upgrade():
     input_params = {
         "template_id": template_id,
         "template_name": broadcast_invitation_template_name,
-        "time_now": datetime.utcnow(),
+        "time_now": utc_now(),
         "content": broadcast_invitation_content,
         "service_id": service_id,
         "subject": broadcast_invitation_subject,

migrations/versions/0347_add_dvla_volumes_template.py

@@ -13,6 +13,8 @@ from alembic import op
 from flask import current_app
 from sqlalchemy import text
 
+from app.utils import utc_now
+
 revision = "0347_add_dvla_volumes_template"
 down_revision = "0346_notify_number_sms_sender"
 
@@ -57,7 +59,7 @@ def upgrade():
         "template_id": email_template_id,
         "template_name": email_template_name,
         "template_type": "email",
-        "time_now": datetime.utcnow(),
+        "time_now": utc_now(),
         "content": email_template_content,
         "notify_service_id": current_app.config["NOTIFY_SERVICE_ID"],
         "subject": email_template_subject,

migrations/versions/0401_add_e2e_test_user.py

@@ -16,6 +16,7 @@ from alembic import op
 from app import db
 from app.dao.users_dao import get_user_by_email
 from app.models import User
+from app.utils import utc_now
 
 revision = "0401_add_e2e_test_user"
 down_revision = "0400_add_total_message_limit"
@@ -32,11 +33,11 @@ def upgrade():
         "password": password,
         "mobile_number": "+12025555555",
         "state": "active",
-        "created_at": datetime.datetime.utcnow(),
-        "password_changed_at": datetime.datetime.utcnow(),
+        "created_at": utc_now(),
+        "password_changed_at": utc_now(),
         "failed_login_count": 0,
         "platform_admin": "f",
-        "email_access_validated_at": datetime.datetime.utcnow(),
+        "email_access_validated_at": utc_now(),
     }
     conn = op.get_bind()
     insert_sql = """

poetry.lock

@@ -204,17 +204,17 @@ tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "p
 
 [[package]]
 name = "awscli"
-version = "1.33.18"
+version = "1.33.21"
 description = "Universal Command Line Environment for AWS."
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "awscli-1.33.18-py3-none-any.whl", hash = "sha256:4065a0c9ee7bd2281e0b04616242693abbe17cd9d7be966abc7a850d5044226d"},
-    {file = "awscli-1.33.18.tar.gz", hash = "sha256:800cae2c020dae7e86877e2b53dee637c19acc62de8084bc67e3434ac174ca35"},
+    {file = "awscli-1.33.21-py3-none-any.whl", hash = "sha256:92e5f8a0f5e3497459f1711ef4044ac1e60ff8d2058e889c63aa6929d5c67459"},
+    {file = "awscli-1.33.21.tar.gz", hash = "sha256:d0a7209e323c85b28d85cffa9470fff664d5f861bb3b5bda843329e0836f5760"},
 ]
 
 [package.dependencies]
-botocore = "1.34.136"
+botocore = "1.34.139"
 colorama = ">=0.2.5,<0.4.7"
 docutils = ">=0.10,<0.17"
 PyYAML = ">=3.10,<6.1"
@@ -403,17 +403,17 @@ files = [
 
 [[package]]
 name = "boto3"
-version = "1.34.136"
+version = "1.34.138"
 description = "The AWS SDK for Python"
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "boto3-1.34.136-py3-none-any.whl", hash = "sha256:d41037e2c680ab8d6c61a0a4ee6bf1fdd9e857f43996672830a95d62d6f6fa79"},
-    {file = "boto3-1.34.136.tar.gz", hash = "sha256:0314e6598f59ee0f34eb4e6d1a0f69fa65c146d2b88a6e837a527a9956ec2731"},
+    {file = "boto3-1.34.138-py3-none-any.whl", hash = "sha256:81518aa95fad71279411fb5c94da4b4a554a5d53fc876faca62b7b5c8737f1cb"},
+    {file = "boto3-1.34.138.tar.gz", hash = "sha256:f79c15e33eb7706f197d98d828b193cf0891966682ad3ec5e900f6f9e7362e35"},
 ]
 
 [package.dependencies]
-botocore = ">=1.34.136,<1.35.0"
+botocore = ">=1.34.138,<1.35.0"
 jmespath = ">=0.7.1,<2.0.0"
 s3transfer = ">=0.10.0,<0.11.0"
 
@@ -422,13 +422,13 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"]
 
 [[package]]
 name = "botocore"
-version = "1.34.136"
+version = "1.34.139"
 description = "Low-level, data-driven core of boto 3."
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "botocore-1.34.136-py3-none-any.whl", hash = "sha256:c63fe9032091fb9e9477706a3ebfa4d0c109b807907051d892ed574f9b573e61"},
-    {file = "botocore-1.34.136.tar.gz", hash = "sha256:7f7135178692b39143c8f152a618d2a3b71065a317569a7102d2306d4946f42f"},
+    {file = "botocore-1.34.139-py3-none-any.whl", hash = "sha256:dd1e085d4caa2a4c1b7d83e3bc51416111c8238a35d498e9d3b04f3b63b086ba"},
+    {file = "botocore-1.34.139.tar.gz", hash = "sha256:df023d8cf8999d574214dad4645cb90f9d2ccd1494f6ee2b57b1ab7522f6be77"},
 ]
 
 [package.dependencies]
@@ -553,13 +553,13 @@ zstd = ["zstandard (==0.22.0)"]
 
 [[package]]
 name = "certifi"
-version = "2024.6.2"
+version = "2024.7.4"
 description = "Python package for providing Mozilla's CA Bundle."
 optional = false
 python-versions = ">=3.6"
 files = [
-    {file = "certifi-2024.6.2-py3-none-any.whl", hash = "sha256:ddc6c8ce995e6987e7faf5e3f1b02b302836a0e5d98ece18392cb1a36c72ad56"},
-    {file = "certifi-2024.6.2.tar.gz", hash = "sha256:3cd43f1c6fa7dedc5899d69d3ad0398fd018ad1a17fba83ddaf78aa46c747516"},
+    {file = "certifi-2024.7.4-py3-none-any.whl", hash = "sha256:c198e21b1289c2ab85ee4e67bb4b4ef3ead0892059901a8d5b622f24a1101e90"},
+    {file = "certifi-2024.7.4.tar.gz", hash = "sha256:5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b"},
 ]
 
 [[package]]
@@ -2098,9 +2098,13 @@ files = [
     {file = "lxml-5.2.2-cp36-cp36m-win_amd64.whl", hash = "sha256:edcfa83e03370032a489430215c1e7783128808fd3e2e0a3225deee278585196"},
     {file = "lxml-5.2.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:28bf95177400066596cdbcfc933312493799382879da504633d16cf60bba735b"},
     {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3a745cc98d504d5bd2c19b10c79c61c7c3df9222629f1b6210c0368177589fb8"},
+    {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1b590b39ef90c6b22ec0be925b211298e810b4856909c8ca60d27ffbca6c12e6"},
     {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b336b0416828022bfd5a2e3083e7f5ba54b96242159f83c7e3eebaec752f1716"},
+    {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_28_aarch64.whl", hash = "sha256:c2faf60c583af0d135e853c86ac2735ce178f0e338a3c7f9ae8f622fd2eb788c"},
     {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:4bc6cb140a7a0ad1f7bc37e018d0ed690b7b6520ade518285dc3171f7a117905"},
+    {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7ff762670cada8e05b32bf1e4dc50b140790909caa8303cfddc4d702b71ea184"},
     {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:57f0a0bbc9868e10ebe874e9f129d2917750adf008fe7b9c1598c0fbbfdde6a6"},
+    {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:a6d2092797b388342c1bc932077ad232f914351932353e2e8706851c870bca1f"},
     {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:60499fe961b21264e17a471ec296dcbf4365fbea611bf9e303ab69db7159ce61"},
     {file = "lxml-5.2.2-cp37-cp37m-win32.whl", hash = "sha256:d9b342c76003c6b9336a80efcc766748a333573abf9350f4094ee46b006ec18f"},
     {file = "lxml-5.2.2-cp37-cp37m-win_amd64.whl", hash = "sha256:b16db2770517b8799c79aa80f4053cd6f8b716f21f8aca962725a9565ce3ee40"},
@@ -2385,13 +2389,13 @@ files = [
 
 [[package]]
 name = "moto"
-version = "5.0.9"
+version = "5.0.10"
 description = ""
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "moto-5.0.9-py2.py3-none-any.whl", hash = "sha256:21a13e02f83d6a18cfcd99949c96abb2e889f4bd51c4c6a3ecc8b78765cb854e"},
-    {file = "moto-5.0.9.tar.gz", hash = "sha256:eb71f1cba01c70fff1f16086acb24d6d9aeb32830d646d8989f98a29aeae24ba"},
+    {file = "moto-5.0.10-py2.py3-none-any.whl", hash = "sha256:9ffae2f64cc8fe95b9a12d63ae7268a7d6bea9993b922905b5abd8197d852cd0"},
+    {file = "moto-5.0.10.tar.gz", hash = "sha256:eff37363221c93ea44f95721ae0ddb56f977fe70437a041b6cc641ee90266279"},
 ]
 
 [package.dependencies]
@@ -2489,7 +2493,6 @@ files = [
     {file = "msgpack-1.0.8-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5fbb160554e319f7b22ecf530a80a3ff496d38e8e07ae763b9e82fadfe96f273"},
     {file = "msgpack-1.0.8-cp39-cp39-win32.whl", hash = "sha256:f9af38a89b6a5c04b7d18c492c8ccf2aee7048aff1ce8437c4683bb5a1df893d"},
     {file = "msgpack-1.0.8-cp39-cp39-win_amd64.whl", hash = "sha256:ed59dd52075f8fc91da6053b12e8c89e37aa043f8986efd89e61fae69dc1b011"},
-    {file = "msgpack-1.0.8-py3-none-any.whl", hash = "sha256:24f727df1e20b9876fa6e95f840a2a2651e34c0ad147676356f4bf5fbb0206ca"},
     {file = "msgpack-1.0.8.tar.gz", hash = "sha256:95c02b0e27e706e48d0e5426d1710ca78e0f0628d6e89d5b5a5b91a5f12274f3"},
 ]
 
@@ -2823,13 +2826,13 @@ ptyprocess = ">=0.5"
 
 [[package]]
 name = "phonenumbers"
-version = "8.13.39"
+version = "8.13.40"
 description = "Python version of Google's common library for parsing, formatting, storing and validating international phone numbers."
 optional = false
 python-versions = "*"
 files = [
-    {file = "phonenumbers-8.13.39-py2.py3-none-any.whl", hash = "sha256:3ad2d086fa71e7eef409001b9195ac54bebb0c6e3e752209b558ca192c9229a0"},
-    {file = "phonenumbers-8.13.39.tar.gz", hash = "sha256:db7ca4970d206b2056231105300753b1a5b229f43416f8c2b3010e63fbb68d77"},
+    {file = "phonenumbers-8.13.40-py2.py3-none-any.whl", hash = "sha256:9582752c20a1da5ec4449f7f97542bf8a793c8e2fec0ab57f767177bb8fc0b1d"},
+    {file = "phonenumbers-8.13.40.tar.gz", hash = "sha256:f137c2848b8e83dd064b71881b65680584417efa202177fd330e2f7ff6c68113"},
 ]
 
 [[package]]
@@ -4017,18 +4020,18 @@ jeepney = ">=0.6"
 
 [[package]]
 name = "setuptools"
-version = "70.1.1"
+version = "70.2.0"
 description = "Easily download, build, install, upgrade, and uninstall Python packages"
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "setuptools-70.1.1-py3-none-any.whl", hash = "sha256:a58a8fde0541dab0419750bcc521fbdf8585f6e5cb41909df3a472ef7b81ca95"},
-    {file = "setuptools-70.1.1.tar.gz", hash = "sha256:937a48c7cdb7a21eb53cd7f9b59e525503aa8abaf3584c730dc5f7a5bec3a650"},
+    {file = "setuptools-70.2.0-py3-none-any.whl", hash = "sha256:b8b8060bb426838fbe942479c90296ce976249451118ef566a5a0b7d8b78fb05"},
+    {file = "setuptools-70.2.0.tar.gz", hash = "sha256:bd63e505105011b25c3c11f753f7e3b8465ea739efddaccef8f0efac2137bac1"},
 ]
 
 [package.extras]
-docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"]
-testing = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test", "mypy (==1.10.0)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.1)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.3.2)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"]
+doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"]
+test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test", "mypy (==1.10.0)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.3.2)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"]
 
 [[package]]
 name = "shapely"
@@ -4748,4 +4751,4 @@ multidict = ">=4.0"
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.12.2"
-content-hash = "d3ca67b44f40fb25b724b8468e07d30901ddced875ffe5d6b6710a17e492b072"
+content-hash = "74d41976bb5028dce7b953ee4c2f6108a46215c8e48d2a4b2152d0277a63b395"

pyproject.toml

@@ -11,8 +11,8 @@ python = "^3.12.2"
 alembic = "==1.13.2"
 amqp = "==5.2.0"
 beautifulsoup4 = "==4.12.3"
-boto3 = "^1.34.136"
-botocore = "^1.34.136"
+boto3 = "^1.34.138"
+botocore = "^1.34.139"
 cachetools = "==5.3.3"
 celery = {version = "==5.4.0", extras = ["redis"]}
 certifi = ">=2022.12.7"
@@ -55,7 +55,7 @@ geojson = "^3.1.0"
 govuk-bank-holidays = "^0.14"
 numpy = "^1.26.4"
 ordered-set = "^4.1.0"
-phonenumbers = "^8.13.39"
+phonenumbers = "^8.13.40"
 python-json-logger = "^2.0.7"
 pytz = "^2024.1"
 regex = "^2024.5.15"
@@ -92,7 +92,7 @@ freezegun = "^1.5.1"
 honcho = "*"
 isort = "^5.13.2"
 jinja2-cli = {version = "==0.8.2", extras = ["yaml"]}
-moto = "==5.0.9"
+moto = "==5.0.10"
 pip-audit = "*"
 pre-commit = "^3.7.1"
 pytest = "^8.2.2"
@@ -102,7 +102,7 @@ pytest-cov = "^5.0.0"
 pytest-xdist = "^3.5.0"
 radon = "^6.0.1"
 requests-mock = "^1.11.0"
-setuptools = "^70.1.1"
+setuptools = "^70.2.0"
 sqlalchemy-utils = "^0.41.2"
 vulture = "^2.10"
 detect-secrets = "^1.5.0"

tests/app/dao/test_fact_notification_status_dao.py

@@ -33,31 +33,44 @@ def test_fetch_notification_status_for_service_by_month(notify_db_session):
     service_1 = create_service(service_name="service_1")
     service_2 = create_service(service_name="service_2")
 
-    create_ft_notification_status(
-        date(2018, 1, 1), NotificationType.SMS, service_1, count=4
-    )
-    create_ft_notification_status(
-        date(2018, 1, 2), NotificationType.SMS, service_1, count=10
-    )
-    create_ft_notification_status(
-        date(2018, 1, 2),
-        NotificationType.SMS,
-        service_1,
-        notification_status=NotificationStatus.CREATED,
-    )
-    create_ft_notification_status(date(2018, 1, 3), NotificationType.EMAIL, service_1)
+    create_template(service=service_1)
+    create_template(service=service_1, template_type=TemplateType.EMAIL)
+    # not the service being tested
+    create_template(service=service_2)
 
-    create_ft_notification_status(date(2018, 2, 2), NotificationType.SMS, service_1)
+    # loop messages for the month
+    for x in range(0, 14):
+        create_notification(
+            service_1.templates[0],
+            created_at=datetime(2018, 1, 1, 1, x, 0),
+            status=NotificationStatus.DELIVERED,
+        )
+    create_notification(
+        service_1.templates[0], created_at=datetime(2018, 1, 1, 1, 1, 0)
+    )
+    create_notification(
+        service_1.templates[1],
+        created_at=datetime(2018, 1, 1, 1, 1, 0),
+        status=NotificationStatus.DELIVERED,
+    )
+    create_notification(
+        service_1.templates[0],
+        created_at=datetime(2018, 2, 1, 1, 1, 0),
+        status=NotificationStatus.DELIVERED,
+    )
 
-    # not included - too early
-    create_ft_notification_status(date(2017, 12, 31), NotificationType.SMS, service_1)
-    # not included - too late
-    create_ft_notification_status(date(2017, 3, 1), NotificationType.SMS, service_1)
-    # not included - wrong service
-    create_ft_notification_status(date(2018, 1, 3), NotificationType.SMS, service_2)
-    # not included - test keys
-    create_ft_notification_status(
-        date(2018, 1, 3), NotificationType.SMS, service_1, key_type=KeyType.TEST
+    # not the right month
+    create_notification(
+        service_1.templates[0],
+        created_at=datetime(2018, 4, 1, 1, 1, 0),
+        status=NotificationStatus.DELIVERED,
+    )
+
+    # not the right service
+    create_notification(
+        service_2.templates[0],
+        created_at=datetime(2018, 2, 1, 1, 1, 0),
+        status=NotificationStatus.DELIVERED,
     )
 
     results = sorted(

tests/app/service/test_statistics.py

@@ -301,12 +301,15 @@ def test_add_monthly_notification_status_stats():
     data = create_empty_monthly_notification_status_stats_dict(2018)
     # this data won't be affected
     data["2018-05"][NotificationType.EMAIL][NotificationStatus.SENDING] = 32
+    data["2018-05"][NotificationType.EMAIL][StatisticsType.REQUESTED] = 32
 
     # this data will get combined with the 8 from row_data
     data["2018-05"][NotificationType.SMS][NotificationStatus.SENDING] = 16
+    data["2018-05"][NotificationType.SMS][StatisticsType.REQUESTED] = 16
 
     add_monthly_notification_status_stats(data, rows)
     # first 3 months are empty
+
     assert data == {
         "2018-01": {NotificationType.SMS: {}, NotificationType.EMAIL: {}},
         "2018-02": {NotificationType.SMS: {}, NotificationType.EMAIL: {}},
@@ -315,12 +318,22 @@ def test_add_monthly_notification_status_stats():
             NotificationType.SMS: {
                 NotificationStatus.SENDING: 1,
                 NotificationStatus.DELIVERED: 2,
+                StatisticsType.REQUESTED: 3,
+            },
+            NotificationType.EMAIL: {
+                NotificationStatus.SENDING: 4,
+                StatisticsType.REQUESTED: 4,
             },
-            NotificationType.EMAIL: {NotificationStatus.SENDING: 4},
         },
         "2018-05": {
-            NotificationType.SMS: {NotificationStatus.SENDING: 24},
-            NotificationType.EMAIL: {NotificationStatus.SENDING: 32},
+            NotificationType.SMS: {
+                NotificationStatus.SENDING: 24,
+                StatisticsType.REQUESTED: 24,
+            },
+            NotificationType.EMAIL: {
+                NotificationStatus.SENDING: 32,
+                StatisticsType.REQUESTED: 32,
+            },
         },
         "2018-06": {NotificationType.SMS: {}, NotificationType.EMAIL: {}},
     }

tests/app/service/test_statistics_rest.py

@@ -234,17 +234,36 @@ def test_get_monthly_notification_stats_returns_stats(admin_request, sample_serv
     sms_t2 = create_template(sample_service)
     email_template = create_template(sample_service, template_type=TemplateType.EMAIL)
 
-    create_ft_notification_status(datetime(2016, 6, 1), template=sms_t1)
-    create_ft_notification_status(datetime(2016, 6, 2), template=sms_t1)
-
-    create_ft_notification_status(datetime(2016, 7, 1), template=sms_t1)
-    create_ft_notification_status(datetime(2016, 7, 1), template=sms_t2)
-    create_ft_notification_status(
-        datetime(2016, 7, 1),
-        template=sms_t1,
-        notification_status=NotificationStatus.CREATED,
+    create_notification(
+        sms_t1,
+        created_at=datetime(2016, 6, 1, 1, 1, 0),
+        status=NotificationStatus.DELIVERED,
+    )
+    create_notification(
+        sms_t1,
+        created_at=datetime(2016, 6, 2, 1, 1, 0),
+        status=NotificationStatus.DELIVERED,
+    )
+    create_notification(
+        sms_t1,
+        created_at=datetime(2016, 7, 1, 1, 1, 0),
+        status=NotificationStatus.DELIVERED,
+    )
+    create_notification(
+        sms_t2,
+        created_at=datetime(2016, 7, 1, 1, 1, 0),
+        status=NotificationStatus.DELIVERED,
+    )
+    create_notification(
+        sms_t1,
+        created_at=datetime(2016, 7, 1, 1, 1, 0),
+        status=NotificationStatus.CREATED,
+    )
+    create_notification(
+        email_template,
+        created_at=datetime(2016, 7, 1, 1, 1, 0),
+        status=NotificationStatus.DELIVERED,
     )
-    create_ft_notification_status(datetime(2016, 7, 1), template=email_template)
 
     response = admin_request.get(
         "service.get_monthly_notification_stats",
@@ -256,7 +275,8 @@ def test_get_monthly_notification_stats_returns_stats(admin_request, sample_serv
     assert response["data"]["2016-06"] == {
         NotificationType.SMS: {
             # it combines the two days
-            NotificationStatus.DELIVERED: 2
+            NotificationStatus.DELIVERED: 2,
+            StatisticsType.REQUESTED: 2,
         },
         NotificationType.EMAIL: {},
     }
@@ -265,86 +285,43 @@ def test_get_monthly_notification_stats_returns_stats(admin_request, sample_serv
         NotificationType.SMS: {
             NotificationStatus.CREATED: 1,
             NotificationStatus.DELIVERED: 2,
+            StatisticsType.REQUESTED: 3,
         },
-        NotificationType.EMAIL: {StatisticsType.DELIVERED: 1},
-    }
-
-
-@freeze_time("2016-06-05 12:00:00")
-def test_get_monthly_notification_stats_combines_todays_data_and_historic_stats(
-    admin_request, sample_template
-):
-    create_ft_notification_status(
-        datetime(2016, 5, 1, 12),
-        template=sample_template,
-        count=1,
-    )
-    create_ft_notification_status(
-        datetime(2016, 6, 1, 12),
-        template=sample_template,
-        notification_status=NotificationStatus.CREATED,
-        count=2,
-    )  # noqa
-
-    create_notification(
-        sample_template,
-        created_at=datetime(2016, 6, 5, 12),
-        status=NotificationStatus.CREATED,
-    )
-    create_notification(
-        sample_template,
-        created_at=datetime(2016, 6, 5, 12),
-        status=NotificationStatus.DELIVERED,
-    )
-
-    # this doesn't get returned in the stats because it is old - it should be in ft_notification_status by now
-    create_notification(
-        sample_template,
-        created_at=datetime(2016, 6, 4, 12),
-        status=NotificationStatus.SENDING,
-    )
-
-    response = admin_request.get(
-        "service.get_monthly_notification_stats",
-        service_id=sample_template.service_id,
-        year=2016,
-    )
-
-    assert len(response["data"]) == 6  # January to June
-    assert response["data"]["2016-05"] == {
-        NotificationType.SMS: {NotificationStatus.DELIVERED: 1},
-        NotificationType.EMAIL: {},
-    }
-    assert response["data"]["2016-06"] == {
-        NotificationType.SMS: {
-            # combines the stats from the historic ft_notification_status and the current notifications
-            NotificationStatus.CREATED: 3,
-            NotificationStatus.DELIVERED: 1,
+        NotificationType.EMAIL: {
+            StatisticsType.DELIVERED: 1,
+            StatisticsType.REQUESTED: 1,
         },
-        NotificationType.EMAIL: {},
     }
 
 
 def test_get_monthly_notification_stats_ignores_test_keys(
     admin_request, sample_service
 ):
-    create_ft_notification_status(
-        datetime(2016, 6, 1),
-        service=sample_service,
+    create_template(service=sample_service)
+
+    create_notification(
+        sample_service.templates[0],
+        created_at=datetime(2016, 6, 1, 1, 1, 0),
         key_type=KeyType.NORMAL,
-        count=1,
+        status=NotificationStatus.DELIVERED,
     )
-    create_ft_notification_status(
-        datetime(2016, 6, 1),
-        service=sample_service,
+    create_notification(
+        sample_service.templates[0],
+        created_at=datetime(2016, 6, 2, 1, 1, 0),
+        key_type=KeyType.NORMAL,
+        status=NotificationStatus.DELIVERED,
+    )
+    create_notification(
+        sample_service.templates[0],
+        created_at=datetime(2016, 6, 1, 1, 1, 0),
         key_type=KeyType.TEAM,
-        count=2,
+        status=NotificationStatus.DELIVERED,
     )
-    create_ft_notification_status(
-        datetime(2016, 6, 1),
-        service=sample_service,
+    create_notification(
+        sample_service.templates[0],
+        created_at=datetime(2016, 6, 1, 1, 1, 0),
         key_type=KeyType.TEST,
-        count=4,
+        status=NotificationStatus.DELIVERED,
     )
 
     response = admin_request.get(
@@ -355,26 +332,27 @@ def test_get_monthly_notification_stats_ignores_test_keys(
 
     assert response["data"]["2016-06"][NotificationType.SMS] == {
         NotificationStatus.DELIVERED: 3,
+        StatisticsType.REQUESTED: 3,
     }
 
 
 def test_get_monthly_notification_stats_checks_dates(admin_request, sample_service):
     t = create_template(sample_service)
-    # create_ft_notification_status(datetime(2016, 3, 31), template=t, notification_status='created')
-    create_ft_notification_status(
-        datetime(2016, 4, 2),
-        template=t,
-        notification_status=NotificationStatus.SENDING,
+
+    create_notification(
+        t,
+        created_at=datetime(2016, 4, 2),
+        status=NotificationStatus.SENDING,
     )
-    create_ft_notification_status(
-        datetime(2017, 3, 31),
-        template=t,
-        notification_status=NotificationStatus.DELIVERED,
+    create_notification(
+        t,
+        created_at=datetime(2017, 3, 31),
+        status=NotificationStatus.DELIVERED,
     )
-    create_ft_notification_status(
-        datetime(2017, 4, 11),
-        template=t,
-        notification_status=NotificationStatus.PERMANENT_FAILURE,
+    create_notification(
+        t,
+        created_at=datetime(2017, 4, 11),
+        status=NotificationStatus.PERMANENT_FAILURE,
     )
 
     response = admin_request.get(
@@ -386,9 +364,11 @@ def test_get_monthly_notification_stats_checks_dates(admin_request, sample_servi
     assert "2017-04" not in response["data"]
     assert response["data"]["2016-04"][NotificationType.SMS] == {
         NotificationStatus.SENDING: 1,
+        StatisticsType.REQUESTED: 1,
     }
     assert response["data"]["2016-04"][NotificationType.SMS] == {
         NotificationStatus.SENDING: 1,
+        StatisticsType.REQUESTED: 1,
     }
 
 
@@ -399,15 +379,15 @@ def test_get_monthly_notification_stats_only_gets_for_one_service(
 
     templates = [create_template(services[0]), create_template(services[1])]
 
-    create_ft_notification_status(
-        datetime(2016, 6, 1),
-        template=templates[0],
-        notification_status=NotificationStatus.CREATED,
+    create_notification(
+        templates[0],
+        created_at=datetime(2016, 6, 1),
+        status=NotificationStatus.CREATED,
     )
-    create_ft_notification_status(
-        datetime(2016, 6, 1),
-        template=templates[1],
-        notification_status=NotificationStatus.DELIVERED,
+    create_notification(
+        templates[1],
+        created_at=datetime(2016, 6, 1),
+        status=NotificationStatus.DELIVERED,
     )
 
     response = admin_request.get(
@@ -417,6 +397,9 @@ def test_get_monthly_notification_stats_only_gets_for_one_service(
     )
 
     assert response["data"]["2016-06"] == {
-        NotificationType.SMS: {NotificationStatus.CREATED: 1},
+        NotificationType.SMS: {
+            NotificationStatus.CREATED: 1,
+            StatisticsType.REQUESTED: 1,
+        },
         NotificationType.EMAIL: {},
     }