Merge pull request #1059 from GSA/1006-new-api-failed-and-delivered-messages-7-days

1006/1007/1008 created new API endpoints for front end statistics

app/dao/date_util.py

@@ -1,3 +1,4 @@
+import calendar
 from datetime import date, datetime, time, timedelta
 
 from app.utils import utc_now
@@ -66,3 +67,29 @@ def get_calendar_year_for_datetime(start_date):
         return year - 1
     else:
         return year
+
+
+def get_number_of_days_for_month(year, month):
+    return calendar.monthrange(year, month)[1]
+
+
+def generate_date_range(start_date, end_date=None, days=0):
+    if end_date:
+        current_date = start_date
+        while current_date <= end_date:
+            try:
+                yield current_date.date()
+            except ValueError:
+                pass
+            current_date += timedelta(days=1)
+    elif days > 0:
+        end_date = start_date + timedelta(days=days)
+        current_date = start_date
+        while current_date < end_date:
+            try:
+                yield current_date.date()
+            except ValueError:
+                pass
+            current_date += timedelta(days=1)
+    else:
+        return "An end_date or number of days must be specified"

app/dao/fact_notification_status_dao.py

@@ -84,21 +84,21 @@ def update_fact_notification_status(process_day, notification_type, service_id):
 def fetch_notification_status_for_service_by_month(start_date, end_date, service_id):
     return (
         db.session.query(
-            func.date_trunc("month", FactNotificationStatus.local_date).label("month"),
+            func.date_trunc("month", NotificationAllTimeView.created_at).label("month"),
-            FactNotificationStatus.notification_type,
+            NotificationAllTimeView.notification_type,
-            FactNotificationStatus.notification_status,
+            NotificationAllTimeView.status.label("notification_status"),
-            func.sum(FactNotificationStatus.notification_count).label("count"),
+            func.count(NotificationAllTimeView.id).label("count"),
         )
         .filter(
-            FactNotificationStatus.service_id == service_id,
+            NotificationAllTimeView.service_id == service_id,
-            FactNotificationStatus.local_date >= start_date,
+            NotificationAllTimeView.created_at >= start_date,
-            FactNotificationStatus.local_date < end_date,
+            NotificationAllTimeView.created_at < end_date,
-            FactNotificationStatus.key_type != KeyType.TEST,
+            NotificationAllTimeView.key_type != KeyType.TEST,
         )
         .group_by(
-            func.date_trunc("month", FactNotificationStatus.local_date).label("month"),
+            func.date_trunc("month", NotificationAllTimeView.created_at).label("month"),
-            FactNotificationStatus.notification_type,
+            NotificationAllTimeView.notification_type,
-            FactNotificationStatus.notification_status,
+            NotificationAllTimeView.status,
         )
         .all()
     )

app/dao/services_dao.py

@@ -8,7 +8,7 @@ from sqlalchemy.sql.expression import and_, asc, case, func
 
 from app import db
 from app.dao.dao_utils import VersionOptions, autocommit, version_class
-from app.dao.date_util import get_current_calendar_year
+from app.dao.date_util import generate_date_range, get_current_calendar_year
 from app.dao.organization_dao import dao_get_organization_by_email_address
 from app.dao.service_sms_sender_dao import insert_service_sms_sender
 from app.dao.service_user_dao import dao_get_service_user
@@ -27,6 +27,7 @@ from app.models import (
     InvitedUser,
     Job,
     Notification,
+    NotificationAllTimeView,
     NotificationHistory,
     Organization,
     Permission,
@@ -40,6 +41,7 @@ from app.models import (
     User,
     VerifyCode,
 )
+from app.service import statistics
 from app.utils import (
     escape_special_characters,
     get_archived_db_column_value,
@@ -426,6 +428,61 @@ def dao_fetch_todays_stats_for_service(service_id):
     )
 
 
+def dao_fetch_stats_for_service_from_days(service_id, start_date, end_date):
+    start_date = get_midnight_in_utc(start_date)
+    end_date = get_midnight_in_utc(end_date + timedelta(days=1))
+
+    return (
+        db.session.query(
+            NotificationAllTimeView.notification_type,
+            NotificationAllTimeView.status,
+            func.date_trunc("day", NotificationAllTimeView.created_at).label("day"),
+            func.count(NotificationAllTimeView.id).label("count"),
+        )
+        .filter(
+            NotificationAllTimeView.service_id == service_id,
+            NotificationAllTimeView.key_type != KeyType.TEST,
+            NotificationAllTimeView.created_at >= start_date,
+            NotificationAllTimeView.created_at < end_date,
+        )
+        .group_by(
+            NotificationAllTimeView.notification_type,
+            NotificationAllTimeView.status,
+            func.date_trunc("day", NotificationAllTimeView.created_at),
+        )
+        .all()
+    )
+
+
+def dao_fetch_stats_for_service_from_days_for_user(
+    service_id, start_date, end_date, user_id
+):
+    start_date = get_midnight_in_utc(start_date)
+    end_date = get_midnight_in_utc(end_date + timedelta(days=1))
+
+    return (
+        db.session.query(
+            NotificationAllTimeView.notification_type,
+            NotificationAllTimeView.status,
+            func.date_trunc("day", NotificationAllTimeView.created_at).label("day"),
+            func.count(NotificationAllTimeView.id).label("count"),
+        )
+        .filter(
+            NotificationAllTimeView.service_id == service_id,
+            NotificationAllTimeView.key_type != KeyType.TEST,
+            NotificationAllTimeView.created_at >= start_date,
+            NotificationAllTimeView.created_at < end_date,
+            NotificationAllTimeView.created_by_id == user_id,
+        )
+        .group_by(
+            NotificationAllTimeView.notification_type,
+            NotificationAllTimeView.status,
+            func.date_trunc("day", NotificationAllTimeView.created_at),
+        )
+        .all()
+    )
+
+
 def dao_fetch_todays_stats_for_all_services(
     include_from_test_key=True, only_active=True
 ):
@@ -607,3 +664,52 @@ def get_live_services_with_organization():
     )
 
     return query.all()
+
+
+def fetch_notification_stats_for_service_by_month_by_user(
+    start_date, end_date, service_id, user_id
+):
+    return (
+        db.session.query(
+            func.date_trunc("month", NotificationAllTimeView.created_at).label("month"),
+            NotificationAllTimeView.notification_type,
+            (NotificationAllTimeView.status).label("notification_status"),
+            func.count(NotificationAllTimeView.id).label("count"),
+        )
+        .filter(
+            NotificationAllTimeView.service_id == service_id,
+            NotificationAllTimeView.created_at >= start_date,
+            NotificationAllTimeView.created_at < end_date,
+            NotificationAllTimeView.key_type != KeyType.TEST,
+            NotificationAllTimeView.created_by_id == user_id,
+        )
+        .group_by(
+            func.date_trunc("month", NotificationAllTimeView.created_at).label("month"),
+            NotificationAllTimeView.notification_type,
+            NotificationAllTimeView.status,
+        )
+        .all()
+    )
+
+
+def get_specific_days_stats(results, start_date, days=None, end_date=None):
+    if days is not None and end_date is not None:
+        raise ValueError("Only set days OR set end_date, not both.")
+    elif days is not None:
+        gen_range = generate_date_range(start_date, days=days)
+    elif end_date is not None:
+        gen_range = generate_date_range(start_date, end_date)
+    else:
+        raise ValueError("Either days or end_date must be set.")
+
+    grouped_results = {date: [] for date in gen_range} | {
+        day.date(): [notification_type, status, day, count]
+        for notification_type, status, day, count in results
+    }
+
+    stats = {
+        day.strftime("%Y-%m-%d"): statistics.format_statistics(rows)
+        for day, rows in grouped_results.items()
+    }
+
+    return stats

app/service/rest.py

@@ -1,5 +1,5 @@
 import itertools
-from datetime import datetime
+from datetime import datetime, timedelta
 
 from flask import Blueprint, current_app, jsonify, request
 from sqlalchemy.exc import IntegrityError
@@ -17,7 +17,7 @@ from app.dao.api_key_dao import (
     save_model_api_key,
 )
 from app.dao.dao_utils import dao_rollback, transaction
-from app.dao.date_util import get_calendar_year
+from app.dao.date_util import get_calendar_year, get_month_start_and_end_date_in_utc
 from app.dao.fact_notification_status_dao import (
     fetch_monthly_template_usage_for_service,
     fetch_notification_status_for_service_by_month,
@@ -63,13 +63,17 @@ from app.dao.services_dao import (
     dao_fetch_all_services_by_user,
     dao_fetch_live_services_data,
     dao_fetch_service_by_id,
+    dao_fetch_stats_for_service_from_days,
+    dao_fetch_stats_for_service_from_days_for_user,
     dao_fetch_todays_stats_for_all_services,
     dao_fetch_todays_stats_for_service,
     dao_remove_user_from_service,
     dao_resume_service,
     dao_suspend_service,
     dao_update_service,
+    fetch_notification_stats_for_service_by_month_by_user,
     get_services_by_partial_name,
+    get_specific_days_stats,
 )
 from app.dao.templates_dao import dao_get_template_by_id
 from app.dao.users_dao import get_user_by_id
@@ -210,6 +214,58 @@ def get_service_notification_statistics(service_id):
     )
 
 
+@service_blueprint.route("/<uuid:service_id>/statistics/<string:start>/<int:days>")
+def get_service_notification_statistics_by_day(service_id, start, days):
+    return jsonify(
+        data=get_service_statistics_for_specific_days(service_id, start, int(days))
+    )
+
+
+def get_service_statistics_for_specific_days(service_id, start, days=1):
+    # start and end dates needs to be reversed because
+    # the end date is today and the start is x days in the past
+    # a day needs to be substracted to allow for today
+    end_date = datetime.strptime(start, "%Y-%m-%d")
+    start_date = end_date - timedelta(days=days - 1)
+
+    results = dao_fetch_stats_for_service_from_days(service_id, start_date, end_date)
+
+    stats = get_specific_days_stats(results, start_date, days=days)
+
+    return stats
+
+
+@service_blueprint.route(
+    "/<uuid:service_id>/statistics/user/<uuid:user_id>/<string:start>/<int:days>"
+)
+def get_service_notification_statistics_by_day_by_user(
+    service_id, user_id, start, days
+):
+    return jsonify(
+        data=get_service_statistics_for_specific_days_by_user(
+            service_id, user_id, start, int(days)
+        )
+    )
+
+
+def get_service_statistics_for_specific_days_by_user(
+    service_id, user_id, start, days=1
+):
+    # start and end dates needs to be reversed because
+    # the end date is today and the start is x days in the past
+    # a day needs to be substracted to allow for today
+    end_date = datetime.strptime(start, "%Y-%m-%d")
+    start_date = end_date - timedelta(days=days - 1)
+
+    results = dao_fetch_stats_for_service_from_days_for_user(
+        service_id, start_date, end_date, user_id
+    )
+
+    stats = get_specific_days_stats(results, start_date, days=days)
+
+    return stats
+
+
 @service_blueprint.route("", methods=["POST"])
 def create_service():
     data = request.get_json()
@@ -592,6 +648,7 @@ def get_monthly_notification_stats(service_id):
     stats = fetch_notification_status_for_service_by_month(
         start_date, end_date, service_id
     )
+
     statistics.add_monthly_notification_status_stats(data, stats)
 
     now = utc_now()
@@ -604,6 +661,87 @@ def get_monthly_notification_stats(service_id):
     return jsonify(data=data)
 
 
+@service_blueprint.route(
+    "/<uuid:service_id>/notifications/<uuid:user_id>/monthly", methods=["GET"]
+)
+def get_monthly_notification_stats_by_user(service_id, user_id):
+    # check service_id validity
+    dao_fetch_service_by_id(service_id)
+    # user = get_user_by_id(user_id=user_id)
+
+    try:
+        year = int(request.args.get("year", "NaN"))
+    except ValueError:
+        raise InvalidRequest("Year must be a number", status_code=400)
+
+    start_date, end_date = get_calendar_year(year)
+
+    data = statistics.create_empty_monthly_notification_status_stats_dict(year)
+
+    stats = fetch_notification_stats_for_service_by_month_by_user(
+        start_date, end_date, service_id, user_id
+    )
+
+    statistics.add_monthly_notification_status_stats(data, stats)
+
+    now = utc_now()
+    if end_date > now:
+        todays_deltas = fetch_notification_status_for_service_for_day(
+            now, service_id=service_id
+        )
+        statistics.add_monthly_notification_status_stats(data, todays_deltas)
+
+    return jsonify(data=data)
+
+
+@service_blueprint.route(
+    "/<uuid:service_id>/notifications/<uuid:user_id>/month", methods=["GET"]
+)
+def get_single_month_notification_stats_by_user(service_id, user_id):
+    # check service_id validity
+    dao_fetch_service_by_id(service_id)
+
+    try:
+        month = int(request.args.get("month", "NaN"))
+        year = int(request.args.get("year", "NaN"))
+    except ValueError:
+        raise InvalidRequest(
+            "Both a month and year are required as numbers", status_code=400
+        )
+
+    month_year = datetime(year, month, 10, 00, 00, 00)
+    start_date, end_date = get_month_start_and_end_date_in_utc(month_year)
+
+    results = dao_fetch_stats_for_service_from_days_for_user(
+        service_id, start_date, end_date, user_id
+    )
+
+    stats = get_specific_days_stats(results, start_date, end_date=end_date)
+    return jsonify(stats)
+
+
+@service_blueprint.route("/<uuid:service_id>/notifications/month", methods=["GET"])
+def get_single_month_notification_stats_for_service(service_id):
+    # check service_id validity
+    dao_fetch_service_by_id(service_id)
+
+    try:
+        month = int(request.args.get("month", "NaN"))
+        year = int(request.args.get("year", "NaN"))
+    except ValueError:
+        raise InvalidRequest(
+            "Both a month and year are required as numbers", status_code=400
+        )
+
+    month_year = datetime(year, month, 10, 00, 00, 00)
+    start_date, end_date = get_month_start_and_end_date_in_utc(month_year)
+
+    results = dao_fetch_stats_for_service_from_days(service_id, start_date, end_date)
+
+    stats = get_specific_days_stats(results, start_date, end_date=end_date)
+    return jsonify(stats)
+
+
 def get_detailed_service(service_id, today_only=False):
     service = dao_fetch_service_by_id(service_id)
 

app/service/statistics.py

@@ -113,7 +113,6 @@ def create_empty_monthly_notification_status_stats_dict(year):
 def add_monthly_notification_status_stats(data, stats):
     for row in stats:
         month = row.month.strftime("%Y-%m")
-
         data[month][row.notification_type][row.notification_status] += row.count
-
+        data[month][row.notification_type][StatisticsType.REQUESTED] += row.count
     return data

migrations/versions/0025_notify_service_data.py

@@ -15,6 +15,7 @@ from alembic import op
 from sqlalchemy import text
 
 from app.hashing import hashpw
+from app.utils import utc_now
 
 revision = "0025_notify_service_data"
 down_revision = "0024_add_research_mode_defaults"
@@ -32,7 +33,7 @@ def upgrade():
                   """
     conn.execute(
         text(user_insert),
-        {"user_id": user_id, "time_now": datetime.utcnow(), "password": password},
+        {"user_id": user_id, "time_now": utc_now(), "password": password},
     )
     service_history_insert = """INSERT INTO services_history (id, name, created_at, active, message_limit, restricted, research_mode, email_from, created_by_id, reply_to_email_address, version)
                         VALUES (:service_id, 'Notify service', :time_now, True, 1000, False, False, 'testsender@dispostable.com',
@@ -41,7 +42,7 @@ def upgrade():
                      """
     conn.execute(
         text(service_history_insert),
-        {"service_id": service_id, "time_now": datetime.utcnow(), "user_id": user_id},
+        {"service_id": service_id, "time_now": utc_now(), "user_id": user_id},
     )
     service_insert = """INSERT INTO services (id, name, created_at, active, message_limit, restricted, research_mode, email_from, created_by_id, reply_to_email_address, version)
                         VALUES (:service_id, 'Notify service', :time_now, True, 1000, False, False, 'testsender@dispostable.com',
@@ -49,7 +50,7 @@ def upgrade():
                     """
     conn.execute(
         text(service_insert),
-        {"service_id": service_id, "time_now": datetime.utcnow(), "user_id": user_id},
+        {"service_id": service_id, "time_now": utc_now(), "user_id": user_id},
     )
     user_to_service_insert = """INSERT INTO user_to_service (user_id, service_id) VALUES (:user_id, :service_id)"""
     conn.execute(
@@ -74,7 +75,7 @@ def upgrade():
             "template_id": uuid.uuid4(),
             "template_name": "Notify email verification code",
             "template_type": "email",
-            "time_now": datetime.utcnow(),
+            "time_now": utc_now(),
             "content": email_verification_content,
             "service_id": service_id,
             "subject": "Confirm GOV.UK Notify registration",
@@ -87,7 +88,7 @@ def upgrade():
             "template_id": "ece42649-22a8-4d06-b87f-d52d5d3f0a27",
             "template_name": "Notify email verification code",
             "template_type": "email",
-            "time_now": datetime.utcnow(),
+            "time_now": utc_now(),
             "content": email_verification_content,
             "service_id": service_id,
             "subject": "Confirm GOV.UK Notify registration",
@@ -107,7 +108,7 @@ def upgrade():
             "template_id": "4f46df42-f795-4cc4-83bb-65ca312f49cc",
             "template_name": "Notify invitation email",
             "template_type": "email",
-            "time_now": datetime.utcnow(),
+            "time_now": utc_now(),
             "content": invitation_content,
             "service_id": service_id,
             "subject": invitation_subject,
@@ -120,7 +121,7 @@ def upgrade():
             "template_id": "4f46df42-f795-4cc4-83bb-65ca312f49cc",
             "template_name": "Notify invitation email",
             "template_type": "email",
-            "time_now": datetime.utcnow(),
+            "time_now": utc_now(),
             "content": invitation_content,
             "service_id": service_id,
             "subject": invitation_subject,
@@ -135,7 +136,7 @@ def upgrade():
             "template_id": "36fb0730-6259-4da1-8a80-c8de22ad4246",
             "template_name": "Notify SMS verify code",
             "template_type": "sms",
-            "time_now": datetime.utcnow(),
+            "time_now": utc_now(),
             "content": sms_code_content,
             "service_id": service_id,
             "subject": None,
@@ -149,7 +150,7 @@ def upgrade():
             "template_id": "36fb0730-6259-4da1-8a80-c8de22ad4246",
             "template_name": "Notify SMS verify code",
             "template_type": "sms",
-            "time_now": datetime.utcnow(),
+            "time_now": utc_now(),
             "content": sms_code_content,
             "service_id": service_id,
             "subject": None,
@@ -172,7 +173,7 @@ def upgrade():
             "template_id": "474e9242-823b-4f99-813d-ed392e7f1201",
             "template_name": "Notify password reset email",
             "template_type": "email",
-            "time_now": datetime.utcnow(),
+            "time_now": utc_now(),
             "content": password_reset_content,
             "service_id": service_id,
             "subject": "Reset your GOV.UK Notify password",
@@ -186,7 +187,7 @@ def upgrade():
             "template_id": "474e9242-823b-4f99-813d-ed392e7f1201",
             "template_name": "Notify password reset email",
             "template_type": "email",
-            "time_now": datetime.utcnow(),
+            "time_now": utc_now(),
             "content": password_reset_content,
             "service_id": service_id,
             "subject": "Reset your GOV.UK Notify password",

migrations/versions/0028_fix_reg_template_history.py

@@ -11,6 +11,8 @@ from datetime import datetime
 
 from sqlalchemy import text
 
+from app.utils import utc_now
+
 revision = "0028_fix_reg_template_history"
 down_revision = "0026_rename_notify_service"
 
@@ -38,7 +40,7 @@ def upgrade():
         "id": "ece42649-22a8-4d06-b87f-d52d5d3f0a27",
         "name": "Notify email verification code",
         "type": "email",
-        "time_now": datetime.utcnow(),
+        "time_now": utc_now(),
         "content": email_verification_content,
         "service_id": service_id,
         "subject": "Confirm GOV.UK Notify registration",

migrations/versions/0082_add_golive_template.py

@@ -14,6 +14,8 @@ from alembic import op
 from flask import current_app
 from sqlalchemy import text
 
+from app.utils import utc_now
+
 revision = "0082_add_go_live_template"
 down_revision = "0081_noti_status_as_enum"
 
@@ -89,7 +91,7 @@ GOV.UK Notify team
         "template_id": template_id,
         "template_name": template_name,
         "template_type": "email",
-        "time_now": datetime.utcnow(),
+        "time_now": utc_now(),
         "content": template_content,
         "notify_service_id": current_app.config["NOTIFY_SERVICE_ID"],
         "subject": template_subject,

migrations/versions/0117_international_sms_notify.py

@@ -9,6 +9,8 @@ Create Date: 2017-08-29 14:09:41.042061
 # revision identifiers, used by Alembic.
 from sqlalchemy import text
 
+from app.utils import utc_now
+
 revision = "0117_international_sms_notify"
 down_revision = "0115_add_inbound_numbers"
 
@@ -22,7 +24,7 @@ NOTIFY_SERVICE_ID = "d6aa2c68-a2d9-4437-ab19-3ae8eb202553"
 def upgrade():
     input_params = {
         "notify_service_id": NOTIFY_SERVICE_ID,
-        "datetime_now": datetime.utcnow(),
+        "datetime_now": utc_now(),
     }
     conn = op.get_bind()
     conn.execute(

migrations/versions/0134_add_email_2fa_template_.py

@@ -12,6 +12,8 @@ from alembic import op
 from flask import current_app
 from sqlalchemy import text
 
+from app.utils import utc_now
+
 revision = "0134_add_email_2fa_template"
 down_revision = "0133_set_services_sms_prefix"
 
@@ -44,7 +46,7 @@ def upgrade():
         "template_id": template_id,
         "template_name": template_name,
         "template_type": "email",
-        "time_now": datetime.utcnow(),
+        "time_now": utc_now(),
         "content": template_content,
         "notify_service_id": current_app.config["NOTIFY_SERVICE_ID"],
         "subject": template_subject,

migrations/versions/0139_migrate_sms_allowance_data.py

@@ -13,6 +13,7 @@ from alembic import op
 from sqlalchemy import text
 
 from app.dao.date_util import get_current_calendar_year_start_year
+from app.utils import utc_now
 
 revision = "0139_migrate_sms_allowance_data"
 down_revision = "0138_sms_sender_nullable"
@@ -34,7 +35,7 @@ def upgrade():
     input_params = {
         "current_year": current_year,
         "default_limit": default_limit,
-        "time_now": datetime.utcnow(),
+        "time_now": utc_now(),
     }
     insert_row_if_not_exist = """
         INSERT INTO annual_billing

migrations/versions/0171_add_org_invite_template.py

@@ -12,6 +12,8 @@ from alembic import op
 from flask import current_app
 from sqlalchemy import text
 
+from app.utils import utc_now
+
 revision = "0171_add_org_invite_template"
 down_revision = "0170_hidden_non_nullable"
 
@@ -53,7 +55,7 @@ def upgrade():
         "template_id": template_id,
         "template_name": template_name,
         "template_type": "email",
-        "time_now": datetime.utcnow(),
+        "time_now": utc_now(),
         "content": template_content,
         "notify_service_id": current_app.config["NOTIFY_SERVICE_ID"],
         "subject": template_subject,

migrations/versions/0265_add_confirm_edit_templates.py

@@ -12,6 +12,8 @@ from alembic import op
 from flask import current_app
 from sqlalchemy import text
 
+from app.utils import utc_now
+
 revision = "0265_add_confirm_edit_templates"
 down_revision = "0264_add_folder_permissions_perm"
 
@@ -57,7 +59,7 @@ def upgrade():
         "template_id": email_template_id,
         "template_name": email_template_name,
         "template_type": "email",
-        "time_now": datetime.utcnow(),
+        "time_now": utc_now(),
         "content": email_template_content,
         "notify_service_id": current_app.config["NOTIFY_SERVICE_ID"],
         "subject": email_template_subject,
@@ -78,7 +80,7 @@ def upgrade():
         "template_id": mobile_template_id,
         "template_name": mobile_template_name,
         "template_type": "sms",
-        "time_now": datetime.utcnow(),
+        "time_now": utc_now(),
         "content": mobile_template_content,
         "notify_service_id": current_app.config["NOTIFY_SERVICE_ID"],
         "subject": None,

migrations/versions/0294_add_verify_reply_to_.py

@@ -12,6 +12,8 @@ from alembic import op
 from flask import current_app
 from sqlalchemy import text
 
+from app.utils import utc_now
+
 revision = "0294_add_verify_reply_to"
 down_revision = "0293_drop_complaint_fk"
 
@@ -58,7 +60,7 @@ def upgrade():
         "template_id": email_template_id,
         "template_name": email_template_name,
         "template_type": "email",
-        "time_now": datetime.utcnow(),
+        "time_now": utc_now(),
         "content": email_template_content,
         "notify_service_id": current_app.config["NOTIFY_SERVICE_ID"],
         "subject": email_template_subject,

migrations/versions/0330_broadcast_invite_email.py

@@ -12,6 +12,8 @@ from datetime import datetime
 from alembic import op
 from sqlalchemy import text
 
+from app.utils import utc_now
+
 revision = "0330_broadcast_invite_email"
 down_revision = "0329_purge_broadcast_data"
 
@@ -60,7 +62,7 @@ def upgrade():
     input_params = {
         "template_id": template_id,
         "template_name": broadcast_invitation_template_name,
-        "time_now": datetime.utcnow(),
+        "time_now": utc_now(),
         "content": broadcast_invitation_content,
         "service_id": service_id,
         "subject": broadcast_invitation_subject,

migrations/versions/0347_add_dvla_volumes_template.py

@@ -13,6 +13,8 @@ from alembic import op
 from flask import current_app
 from sqlalchemy import text
 
+from app.utils import utc_now
+
 revision = "0347_add_dvla_volumes_template"
 down_revision = "0346_notify_number_sms_sender"
 
@@ -57,7 +59,7 @@ def upgrade():
         "template_id": email_template_id,
         "template_name": email_template_name,
         "template_type": "email",
-        "time_now": datetime.utcnow(),
+        "time_now": utc_now(),
         "content": email_template_content,
         "notify_service_id": current_app.config["NOTIFY_SERVICE_ID"],
         "subject": email_template_subject,

migrations/versions/0401_add_e2e_test_user.py

@@ -16,6 +16,7 @@ from alembic import op
 from app import db
 from app.dao.users_dao import get_user_by_email
 from app.models import User
+from app.utils import utc_now
 
 revision = "0401_add_e2e_test_user"
 down_revision = "0400_add_total_message_limit"
@@ -32,11 +33,11 @@ def upgrade():
         "password": password,
         "mobile_number": "+12025555555",
         "state": "active",
-        "created_at": datetime.datetime.utcnow(),
+        "created_at": utc_now(),
-        "password_changed_at": datetime.datetime.utcnow(),
+        "password_changed_at": utc_now(),
         "failed_login_count": 0,
         "platform_admin": "f",
-        "email_access_validated_at": datetime.datetime.utcnow(),
+        "email_access_validated_at": utc_now(),
     }
     conn = op.get_bind()
     insert_sql = """

tests/app/dao/test_fact_notification_status_dao.py

@@ -33,31 +33,44 @@ def test_fetch_notification_status_for_service_by_month(notify_db_session):
     service_1 = create_service(service_name="service_1")
     service_2 = create_service(service_name="service_2")
 
-    create_ft_notification_status(
+    create_template(service=service_1)
-        date(2018, 1, 1), NotificationType.SMS, service_1, count=4
+    create_template(service=service_1, template_type=TemplateType.EMAIL)
-    )
+    # not the service being tested
-    create_ft_notification_status(
+    create_template(service=service_2)
-        date(2018, 1, 2), NotificationType.SMS, service_1, count=10
-    )
-    create_ft_notification_status(
-        date(2018, 1, 2),
-        NotificationType.SMS,
-        service_1,
-        notification_status=NotificationStatus.CREATED,
-    )
-    create_ft_notification_status(date(2018, 1, 3), NotificationType.EMAIL, service_1)
 
-    create_ft_notification_status(date(2018, 2, 2), NotificationType.SMS, service_1)
+    # loop messages for the month
+    for x in range(0, 14):
+        create_notification(
+            service_1.templates[0],
+            created_at=datetime(2018, 1, 1, 1, x, 0),
+            status=NotificationStatus.DELIVERED,
+        )
+    create_notification(
+        service_1.templates[0], created_at=datetime(2018, 1, 1, 1, 1, 0)
+    )
+    create_notification(
+        service_1.templates[1],
+        created_at=datetime(2018, 1, 1, 1, 1, 0),
+        status=NotificationStatus.DELIVERED,
+    )
+    create_notification(
+        service_1.templates[0],
+        created_at=datetime(2018, 2, 1, 1, 1, 0),
+        status=NotificationStatus.DELIVERED,
+    )
 
-    # not included - too early
+    # not the right month
-    create_ft_notification_status(date(2017, 12, 31), NotificationType.SMS, service_1)
+    create_notification(
-    # not included - too late
+        service_1.templates[0],
-    create_ft_notification_status(date(2017, 3, 1), NotificationType.SMS, service_1)
+        created_at=datetime(2018, 4, 1, 1, 1, 0),
-    # not included - wrong service
+        status=NotificationStatus.DELIVERED,
-    create_ft_notification_status(date(2018, 1, 3), NotificationType.SMS, service_2)
+    )
-    # not included - test keys
+
-    create_ft_notification_status(
+    # not the right service
-        date(2018, 1, 3), NotificationType.SMS, service_1, key_type=KeyType.TEST
+    create_notification(
+        service_2.templates[0],
+        created_at=datetime(2018, 2, 1, 1, 1, 0),
+        status=NotificationStatus.DELIVERED,
     )
 
     results = sorted(

tests/app/service/test_statistics.py

@@ -301,12 +301,15 @@ def test_add_monthly_notification_status_stats():
     data = create_empty_monthly_notification_status_stats_dict(2018)
     # this data won't be affected
     data["2018-05"][NotificationType.EMAIL][NotificationStatus.SENDING] = 32
+    data["2018-05"][NotificationType.EMAIL][StatisticsType.REQUESTED] = 32
 
     # this data will get combined with the 8 from row_data
     data["2018-05"][NotificationType.SMS][NotificationStatus.SENDING] = 16
+    data["2018-05"][NotificationType.SMS][StatisticsType.REQUESTED] = 16
 
     add_monthly_notification_status_stats(data, rows)
     # first 3 months are empty
+
     assert data == {
         "2018-01": {NotificationType.SMS: {}, NotificationType.EMAIL: {}},
         "2018-02": {NotificationType.SMS: {}, NotificationType.EMAIL: {}},
@@ -315,12 +318,22 @@ def test_add_monthly_notification_status_stats():
             NotificationType.SMS: {
                 NotificationStatus.SENDING: 1,
                 NotificationStatus.DELIVERED: 2,
+                StatisticsType.REQUESTED: 3,
+            },
+            NotificationType.EMAIL: {
+                NotificationStatus.SENDING: 4,
+                StatisticsType.REQUESTED: 4,
             },
-            NotificationType.EMAIL: {NotificationStatus.SENDING: 4},
         },
         "2018-05": {
-            NotificationType.SMS: {NotificationStatus.SENDING: 24},
+            NotificationType.SMS: {
-            NotificationType.EMAIL: {NotificationStatus.SENDING: 32},
+                NotificationStatus.SENDING: 24,
+                StatisticsType.REQUESTED: 24,
+            },
+            NotificationType.EMAIL: {
+                NotificationStatus.SENDING: 32,
+                StatisticsType.REQUESTED: 32,
+            },
         },
         "2018-06": {NotificationType.SMS: {}, NotificationType.EMAIL: {}},
     }

tests/app/service/test_statistics_rest.py

@@ -234,17 +234,36 @@ def test_get_monthly_notification_stats_returns_stats(admin_request, sample_serv
     sms_t2 = create_template(sample_service)
     email_template = create_template(sample_service, template_type=TemplateType.EMAIL)
 
-    create_ft_notification_status(datetime(2016, 6, 1), template=sms_t1)
+    create_notification(
-    create_ft_notification_status(datetime(2016, 6, 2), template=sms_t1)
+        sms_t1,
-
+        created_at=datetime(2016, 6, 1, 1, 1, 0),
-    create_ft_notification_status(datetime(2016, 7, 1), template=sms_t1)
+        status=NotificationStatus.DELIVERED,
-    create_ft_notification_status(datetime(2016, 7, 1), template=sms_t2)
+    )
-    create_ft_notification_status(
+    create_notification(
-        datetime(2016, 7, 1),
+        sms_t1,
-        template=sms_t1,
+        created_at=datetime(2016, 6, 2, 1, 1, 0),
-        notification_status=NotificationStatus.CREATED,
+        status=NotificationStatus.DELIVERED,
+    )
+    create_notification(
+        sms_t1,
+        created_at=datetime(2016, 7, 1, 1, 1, 0),
+        status=NotificationStatus.DELIVERED,
+    )
+    create_notification(
+        sms_t2,
+        created_at=datetime(2016, 7, 1, 1, 1, 0),
+        status=NotificationStatus.DELIVERED,
+    )
+    create_notification(
+        sms_t1,
+        created_at=datetime(2016, 7, 1, 1, 1, 0),
+        status=NotificationStatus.CREATED,
+    )
+    create_notification(
+        email_template,
+        created_at=datetime(2016, 7, 1, 1, 1, 0),
+        status=NotificationStatus.DELIVERED,
     )
-    create_ft_notification_status(datetime(2016, 7, 1), template=email_template)
 
     response = admin_request.get(
         "service.get_monthly_notification_stats",
@@ -256,7 +275,8 @@ def test_get_monthly_notification_stats_returns_stats(admin_request, sample_serv
     assert response["data"]["2016-06"] == {
         NotificationType.SMS: {
             # it combines the two days
-            NotificationStatus.DELIVERED: 2
+            NotificationStatus.DELIVERED: 2,
+            StatisticsType.REQUESTED: 2,
         },
         NotificationType.EMAIL: {},
     }
@@ -265,86 +285,43 @@ def test_get_monthly_notification_stats_returns_stats(admin_request, sample_serv
         NotificationType.SMS: {
             NotificationStatus.CREATED: 1,
             NotificationStatus.DELIVERED: 2,
+            StatisticsType.REQUESTED: 3,
         },
-        NotificationType.EMAIL: {StatisticsType.DELIVERED: 1},
+        NotificationType.EMAIL: {
-    }
+            StatisticsType.DELIVERED: 1,
-
+            StatisticsType.REQUESTED: 1,
-
-@freeze_time("2016-06-05 12:00:00")
-def test_get_monthly_notification_stats_combines_todays_data_and_historic_stats(
-    admin_request, sample_template
-):
-    create_ft_notification_status(
-        datetime(2016, 5, 1, 12),
-        template=sample_template,
-        count=1,
-    )
-    create_ft_notification_status(
-        datetime(2016, 6, 1, 12),
-        template=sample_template,
-        notification_status=NotificationStatus.CREATED,
-        count=2,
-    )  # noqa
-
-    create_notification(
-        sample_template,
-        created_at=datetime(2016, 6, 5, 12),
-        status=NotificationStatus.CREATED,
-    )
-    create_notification(
-        sample_template,
-        created_at=datetime(2016, 6, 5, 12),
-        status=NotificationStatus.DELIVERED,
-    )
-
-    # this doesn't get returned in the stats because it is old - it should be in ft_notification_status by now
-    create_notification(
-        sample_template,
-        created_at=datetime(2016, 6, 4, 12),
-        status=NotificationStatus.SENDING,
-    )
-
-    response = admin_request.get(
-        "service.get_monthly_notification_stats",
-        service_id=sample_template.service_id,
-        year=2016,
-    )
-
-    assert len(response["data"]) == 6  # January to June
-    assert response["data"]["2016-05"] == {
-        NotificationType.SMS: {NotificationStatus.DELIVERED: 1},
-        NotificationType.EMAIL: {},
-    }
-    assert response["data"]["2016-06"] == {
-        NotificationType.SMS: {
-            # combines the stats from the historic ft_notification_status and the current notifications
-            NotificationStatus.CREATED: 3,
-            NotificationStatus.DELIVERED: 1,
         },
-        NotificationType.EMAIL: {},
     }
 
 
 def test_get_monthly_notification_stats_ignores_test_keys(
     admin_request, sample_service
 ):
-    create_ft_notification_status(
+    create_template(service=sample_service)
-        datetime(2016, 6, 1),
+
-        service=sample_service,
+    create_notification(
+        sample_service.templates[0],
+        created_at=datetime(2016, 6, 1, 1, 1, 0),
         key_type=KeyType.NORMAL,
-        count=1,
+        status=NotificationStatus.DELIVERED,
     )
-    create_ft_notification_status(
+    create_notification(
-        datetime(2016, 6, 1),
+        sample_service.templates[0],
-        service=sample_service,
+        created_at=datetime(2016, 6, 2, 1, 1, 0),
+        key_type=KeyType.NORMAL,
+        status=NotificationStatus.DELIVERED,
+    )
+    create_notification(
+        sample_service.templates[0],
+        created_at=datetime(2016, 6, 1, 1, 1, 0),
         key_type=KeyType.TEAM,
-        count=2,
+        status=NotificationStatus.DELIVERED,
     )
-    create_ft_notification_status(
+    create_notification(
-        datetime(2016, 6, 1),
+        sample_service.templates[0],
-        service=sample_service,
+        created_at=datetime(2016, 6, 1, 1, 1, 0),
         key_type=KeyType.TEST,
-        count=4,
+        status=NotificationStatus.DELIVERED,
     )
 
     response = admin_request.get(
@@ -355,26 +332,27 @@ def test_get_monthly_notification_stats_ignores_test_keys(
 
     assert response["data"]["2016-06"][NotificationType.SMS] == {
         NotificationStatus.DELIVERED: 3,
+        StatisticsType.REQUESTED: 3,
     }
 
 
 def test_get_monthly_notification_stats_checks_dates(admin_request, sample_service):
     t = create_template(sample_service)
-    # create_ft_notification_status(datetime(2016, 3, 31), template=t, notification_status='created')
+
-    create_ft_notification_status(
+    create_notification(
-        datetime(2016, 4, 2),
+        t,
-        template=t,
+        created_at=datetime(2016, 4, 2),
-        notification_status=NotificationStatus.SENDING,
+        status=NotificationStatus.SENDING,
     )
-    create_ft_notification_status(
+    create_notification(
-        datetime(2017, 3, 31),
+        t,
-        template=t,
+        created_at=datetime(2017, 3, 31),
-        notification_status=NotificationStatus.DELIVERED,
+        status=NotificationStatus.DELIVERED,
     )
-    create_ft_notification_status(
+    create_notification(
-        datetime(2017, 4, 11),
+        t,
-        template=t,
+        created_at=datetime(2017, 4, 11),
-        notification_status=NotificationStatus.PERMANENT_FAILURE,
+        status=NotificationStatus.PERMANENT_FAILURE,
     )
 
     response = admin_request.get(
@@ -386,9 +364,11 @@ def test_get_monthly_notification_stats_checks_dates(admin_request, sample_servi
     assert "2017-04" not in response["data"]
     assert response["data"]["2016-04"][NotificationType.SMS] == {
         NotificationStatus.SENDING: 1,
+        StatisticsType.REQUESTED: 1,
     }
     assert response["data"]["2016-04"][NotificationType.SMS] == {
         NotificationStatus.SENDING: 1,
+        StatisticsType.REQUESTED: 1,
     }
 
 
@@ -399,15 +379,15 @@ def test_get_monthly_notification_stats_only_gets_for_one_service(
 
     templates = [create_template(services[0]), create_template(services[1])]
 
-    create_ft_notification_status(
+    create_notification(
-        datetime(2016, 6, 1),
+        templates[0],
-        template=templates[0],
+        created_at=datetime(2016, 6, 1),
-        notification_status=NotificationStatus.CREATED,
+        status=NotificationStatus.CREATED,
     )
-    create_ft_notification_status(
+    create_notification(
-        datetime(2016, 6, 1),
+        templates[1],
-        template=templates[1],
+        created_at=datetime(2016, 6, 1),
-        notification_status=NotificationStatus.DELIVERED,
+        status=NotificationStatus.DELIVERED,
     )
 
     response = admin_request.get(
@@ -417,6 +397,9 @@ def test_get_monthly_notification_stats_only_gets_for_one_service(
     )
 
     assert response["data"]["2016-06"] == {
-        NotificationType.SMS: {NotificationStatus.CREATED: 1},
+        NotificationType.SMS: {
+            NotificationStatus.CREATED: 1,
+            StatisticsType.REQUESTED: 1,
+        },
         NotificationType.EMAIL: {},
     }