diff --git a/app/dao/date_util.py b/app/dao/date_util.py index 98c0372c5..d93986b45 100644 --- a/app/dao/date_util.py +++ b/app/dao/date_util.py @@ -1,3 +1,4 @@ +import calendar from datetime import date, datetime, time, timedelta from app.utils import utc_now @@ -66,3 +67,29 @@ def get_calendar_year_for_datetime(start_date): return year - 1 else: return year + + +def get_number_of_days_for_month(year, month): + return calendar.monthrange(year, month)[1] + + +def generate_date_range(start_date, end_date=None, days=0): + if end_date: + current_date = start_date + while current_date <= end_date: + try: + yield current_date.date() + except ValueError: + pass + current_date += timedelta(days=1) + elif days > 0: + end_date = start_date + timedelta(days=days) + current_date = start_date + while current_date < end_date: + try: + yield current_date.date() + except ValueError: + pass + current_date += timedelta(days=1) + else: + return "An end_date or number of days must be specified" diff --git a/app/dao/fact_notification_status_dao.py b/app/dao/fact_notification_status_dao.py index 22c87fe83..a810ff0db 100644 --- a/app/dao/fact_notification_status_dao.py +++ b/app/dao/fact_notification_status_dao.py @@ -84,21 +84,21 @@ def update_fact_notification_status(process_day, notification_type, service_id): def fetch_notification_status_for_service_by_month(start_date, end_date, service_id): return ( db.session.query( - func.date_trunc("month", FactNotificationStatus.local_date).label("month"), - FactNotificationStatus.notification_type, - FactNotificationStatus.notification_status, - func.sum(FactNotificationStatus.notification_count).label("count"), + func.date_trunc("month", NotificationAllTimeView.created_at).label("month"), + NotificationAllTimeView.notification_type, + NotificationAllTimeView.status.label("notification_status"), + func.count(NotificationAllTimeView.id).label("count"), ) .filter( - FactNotificationStatus.service_id == service_id, - FactNotificationStatus.local_date >= start_date, - FactNotificationStatus.local_date < end_date, - FactNotificationStatus.key_type != KeyType.TEST, + NotificationAllTimeView.service_id == service_id, + NotificationAllTimeView.created_at >= start_date, + NotificationAllTimeView.created_at < end_date, + NotificationAllTimeView.key_type != KeyType.TEST, ) .group_by( - func.date_trunc("month", FactNotificationStatus.local_date).label("month"), - FactNotificationStatus.notification_type, - FactNotificationStatus.notification_status, + func.date_trunc("month", NotificationAllTimeView.created_at).label("month"), + NotificationAllTimeView.notification_type, + NotificationAllTimeView.status, ) .all() ) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 7ddc456e7..6fa663341 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -8,7 +8,7 @@ from sqlalchemy.sql.expression import and_, asc, case, func from app import db from app.dao.dao_utils import VersionOptions, autocommit, version_class -from app.dao.date_util import get_current_calendar_year +from app.dao.date_util import generate_date_range, get_current_calendar_year from app.dao.organization_dao import dao_get_organization_by_email_address from app.dao.service_sms_sender_dao import insert_service_sms_sender from app.dao.service_user_dao import dao_get_service_user @@ -27,6 +27,7 @@ from app.models import ( InvitedUser, Job, Notification, + NotificationAllTimeView, NotificationHistory, Organization, Permission, @@ -40,6 +41,7 @@ from app.models import ( User, VerifyCode, ) +from app.service import statistics from app.utils import ( escape_special_characters, get_archived_db_column_value, @@ -426,6 +428,61 @@ def dao_fetch_todays_stats_for_service(service_id): ) +def dao_fetch_stats_for_service_from_days(service_id, start_date, end_date): + start_date = get_midnight_in_utc(start_date) + end_date = get_midnight_in_utc(end_date + timedelta(days=1)) + + return ( + db.session.query( + NotificationAllTimeView.notification_type, + NotificationAllTimeView.status, + func.date_trunc("day", NotificationAllTimeView.created_at).label("day"), + func.count(NotificationAllTimeView.id).label("count"), + ) + .filter( + NotificationAllTimeView.service_id == service_id, + NotificationAllTimeView.key_type != KeyType.TEST, + NotificationAllTimeView.created_at >= start_date, + NotificationAllTimeView.created_at < end_date, + ) + .group_by( + NotificationAllTimeView.notification_type, + NotificationAllTimeView.status, + func.date_trunc("day", NotificationAllTimeView.created_at), + ) + .all() + ) + + +def dao_fetch_stats_for_service_from_days_for_user( + service_id, start_date, end_date, user_id +): + start_date = get_midnight_in_utc(start_date) + end_date = get_midnight_in_utc(end_date + timedelta(days=1)) + + return ( + db.session.query( + NotificationAllTimeView.notification_type, + NotificationAllTimeView.status, + func.date_trunc("day", NotificationAllTimeView.created_at).label("day"), + func.count(NotificationAllTimeView.id).label("count"), + ) + .filter( + NotificationAllTimeView.service_id == service_id, + NotificationAllTimeView.key_type != KeyType.TEST, + NotificationAllTimeView.created_at >= start_date, + NotificationAllTimeView.created_at < end_date, + NotificationAllTimeView.created_by_id == user_id, + ) + .group_by( + NotificationAllTimeView.notification_type, + NotificationAllTimeView.status, + func.date_trunc("day", NotificationAllTimeView.created_at), + ) + .all() + ) + + def dao_fetch_todays_stats_for_all_services( include_from_test_key=True, only_active=True ): @@ -607,3 +664,52 @@ def get_live_services_with_organization(): ) return query.all() + + +def fetch_notification_stats_for_service_by_month_by_user( + start_date, end_date, service_id, user_id +): + return ( + db.session.query( + func.date_trunc("month", NotificationAllTimeView.created_at).label("month"), + NotificationAllTimeView.notification_type, + (NotificationAllTimeView.status).label("notification_status"), + func.count(NotificationAllTimeView.id).label("count"), + ) + .filter( + NotificationAllTimeView.service_id == service_id, + NotificationAllTimeView.created_at >= start_date, + NotificationAllTimeView.created_at < end_date, + NotificationAllTimeView.key_type != KeyType.TEST, + NotificationAllTimeView.created_by_id == user_id, + ) + .group_by( + func.date_trunc("month", NotificationAllTimeView.created_at).label("month"), + NotificationAllTimeView.notification_type, + NotificationAllTimeView.status, + ) + .all() + ) + + +def get_specific_days_stats(results, start_date, days=None, end_date=None): + if days is not None and end_date is not None: + raise ValueError("Only set days OR set end_date, not both.") + elif days is not None: + gen_range = generate_date_range(start_date, days=days) + elif end_date is not None: + gen_range = generate_date_range(start_date, end_date) + else: + raise ValueError("Either days or end_date must be set.") + + grouped_results = {date: [] for date in gen_range} | { + day.date(): [notification_type, status, day, count] + for notification_type, status, day, count in results + } + + stats = { + day.strftime("%Y-%m-%d"): statistics.format_statistics(rows) + for day, rows in grouped_results.items() + } + + return stats diff --git a/app/service/rest.py b/app/service/rest.py index 0503c71d5..71faab7a1 100644 --- a/app/service/rest.py +++ b/app/service/rest.py @@ -1,5 +1,5 @@ import itertools -from datetime import datetime +from datetime import datetime, timedelta from flask import Blueprint, current_app, jsonify, request from sqlalchemy.exc import IntegrityError @@ -17,7 +17,7 @@ from app.dao.api_key_dao import ( save_model_api_key, ) from app.dao.dao_utils import dao_rollback, transaction -from app.dao.date_util import get_calendar_year +from app.dao.date_util import get_calendar_year, get_month_start_and_end_date_in_utc from app.dao.fact_notification_status_dao import ( fetch_monthly_template_usage_for_service, fetch_notification_status_for_service_by_month, @@ -63,13 +63,17 @@ from app.dao.services_dao import ( dao_fetch_all_services_by_user, dao_fetch_live_services_data, dao_fetch_service_by_id, + dao_fetch_stats_for_service_from_days, + dao_fetch_stats_for_service_from_days_for_user, dao_fetch_todays_stats_for_all_services, dao_fetch_todays_stats_for_service, dao_remove_user_from_service, dao_resume_service, dao_suspend_service, dao_update_service, + fetch_notification_stats_for_service_by_month_by_user, get_services_by_partial_name, + get_specific_days_stats, ) from app.dao.templates_dao import dao_get_template_by_id from app.dao.users_dao import get_user_by_id @@ -210,6 +214,58 @@ def get_service_notification_statistics(service_id): ) +@service_blueprint.route("//statistics//") +def get_service_notification_statistics_by_day(service_id, start, days): + return jsonify( + data=get_service_statistics_for_specific_days(service_id, start, int(days)) + ) + + +def get_service_statistics_for_specific_days(service_id, start, days=1): + # start and end dates needs to be reversed because + # the end date is today and the start is x days in the past + # a day needs to be substracted to allow for today + end_date = datetime.strptime(start, "%Y-%m-%d") + start_date = end_date - timedelta(days=days - 1) + + results = dao_fetch_stats_for_service_from_days(service_id, start_date, end_date) + + stats = get_specific_days_stats(results, start_date, days=days) + + return stats + + +@service_blueprint.route( + "//statistics/user///" +) +def get_service_notification_statistics_by_day_by_user( + service_id, user_id, start, days +): + return jsonify( + data=get_service_statistics_for_specific_days_by_user( + service_id, user_id, start, int(days) + ) + ) + + +def get_service_statistics_for_specific_days_by_user( + service_id, user_id, start, days=1 +): + # start and end dates needs to be reversed because + # the end date is today and the start is x days in the past + # a day needs to be substracted to allow for today + end_date = datetime.strptime(start, "%Y-%m-%d") + start_date = end_date - timedelta(days=days - 1) + + results = dao_fetch_stats_for_service_from_days_for_user( + service_id, start_date, end_date, user_id + ) + + stats = get_specific_days_stats(results, start_date, days=days) + + return stats + + @service_blueprint.route("", methods=["POST"]) def create_service(): data = request.get_json() @@ -592,6 +648,7 @@ def get_monthly_notification_stats(service_id): stats = fetch_notification_status_for_service_by_month( start_date, end_date, service_id ) + statistics.add_monthly_notification_status_stats(data, stats) now = utc_now() @@ -604,6 +661,87 @@ def get_monthly_notification_stats(service_id): return jsonify(data=data) +@service_blueprint.route( + "//notifications//monthly", methods=["GET"] +) +def get_monthly_notification_stats_by_user(service_id, user_id): + # check service_id validity + dao_fetch_service_by_id(service_id) + # user = get_user_by_id(user_id=user_id) + + try: + year = int(request.args.get("year", "NaN")) + except ValueError: + raise InvalidRequest("Year must be a number", status_code=400) + + start_date, end_date = get_calendar_year(year) + + data = statistics.create_empty_monthly_notification_status_stats_dict(year) + + stats = fetch_notification_stats_for_service_by_month_by_user( + start_date, end_date, service_id, user_id + ) + + statistics.add_monthly_notification_status_stats(data, stats) + + now = utc_now() + if end_date > now: + todays_deltas = fetch_notification_status_for_service_for_day( + now, service_id=service_id + ) + statistics.add_monthly_notification_status_stats(data, todays_deltas) + + return jsonify(data=data) + + +@service_blueprint.route( + "//notifications//month", methods=["GET"] +) +def get_single_month_notification_stats_by_user(service_id, user_id): + # check service_id validity + dao_fetch_service_by_id(service_id) + + try: + month = int(request.args.get("month", "NaN")) + year = int(request.args.get("year", "NaN")) + except ValueError: + raise InvalidRequest( + "Both a month and year are required as numbers", status_code=400 + ) + + month_year = datetime(year, month, 10, 00, 00, 00) + start_date, end_date = get_month_start_and_end_date_in_utc(month_year) + + results = dao_fetch_stats_for_service_from_days_for_user( + service_id, start_date, end_date, user_id + ) + + stats = get_specific_days_stats(results, start_date, end_date=end_date) + return jsonify(stats) + + +@service_blueprint.route("//notifications/month", methods=["GET"]) +def get_single_month_notification_stats_for_service(service_id): + # check service_id validity + dao_fetch_service_by_id(service_id) + + try: + month = int(request.args.get("month", "NaN")) + year = int(request.args.get("year", "NaN")) + except ValueError: + raise InvalidRequest( + "Both a month and year are required as numbers", status_code=400 + ) + + month_year = datetime(year, month, 10, 00, 00, 00) + start_date, end_date = get_month_start_and_end_date_in_utc(month_year) + + results = dao_fetch_stats_for_service_from_days(service_id, start_date, end_date) + + stats = get_specific_days_stats(results, start_date, end_date=end_date) + return jsonify(stats) + + def get_detailed_service(service_id, today_only=False): service = dao_fetch_service_by_id(service_id) diff --git a/app/service/statistics.py b/app/service/statistics.py index 90b933960..a6b58e067 100644 --- a/app/service/statistics.py +++ b/app/service/statistics.py @@ -113,7 +113,6 @@ def create_empty_monthly_notification_status_stats_dict(year): def add_monthly_notification_status_stats(data, stats): for row in stats: month = row.month.strftime("%Y-%m") - data[month][row.notification_type][row.notification_status] += row.count - + data[month][row.notification_type][StatisticsType.REQUESTED] += row.count return data diff --git a/migrations/versions/0025_notify_service_data.py b/migrations/versions/0025_notify_service_data.py index 0683e7dd2..e90d01aad 100644 --- a/migrations/versions/0025_notify_service_data.py +++ b/migrations/versions/0025_notify_service_data.py @@ -15,6 +15,7 @@ from alembic import op from sqlalchemy import text from app.hashing import hashpw +from app.utils import utc_now revision = "0025_notify_service_data" down_revision = "0024_add_research_mode_defaults" @@ -32,7 +33,7 @@ def upgrade(): """ conn.execute( text(user_insert), - {"user_id": user_id, "time_now": datetime.utcnow(), "password": password}, + {"user_id": user_id, "time_now": utc_now(), "password": password}, ) service_history_insert = """INSERT INTO services_history (id, name, created_at, active, message_limit, restricted, research_mode, email_from, created_by_id, reply_to_email_address, version) VALUES (:service_id, 'Notify service', :time_now, True, 1000, False, False, 'testsender@dispostable.com', @@ -41,7 +42,7 @@ def upgrade(): """ conn.execute( text(service_history_insert), - {"service_id": service_id, "time_now": datetime.utcnow(), "user_id": user_id}, + {"service_id": service_id, "time_now": utc_now(), "user_id": user_id}, ) service_insert = """INSERT INTO services (id, name, created_at, active, message_limit, restricted, research_mode, email_from, created_by_id, reply_to_email_address, version) VALUES (:service_id, 'Notify service', :time_now, True, 1000, False, False, 'testsender@dispostable.com', @@ -49,7 +50,7 @@ def upgrade(): """ conn.execute( text(service_insert), - {"service_id": service_id, "time_now": datetime.utcnow(), "user_id": user_id}, + {"service_id": service_id, "time_now": utc_now(), "user_id": user_id}, ) user_to_service_insert = """INSERT INTO user_to_service (user_id, service_id) VALUES (:user_id, :service_id)""" conn.execute( @@ -74,7 +75,7 @@ def upgrade(): "template_id": uuid.uuid4(), "template_name": "Notify email verification code", "template_type": "email", - "time_now": datetime.utcnow(), + "time_now": utc_now(), "content": email_verification_content, "service_id": service_id, "subject": "Confirm GOV.UK Notify registration", @@ -87,7 +88,7 @@ def upgrade(): "template_id": "ece42649-22a8-4d06-b87f-d52d5d3f0a27", "template_name": "Notify email verification code", "template_type": "email", - "time_now": datetime.utcnow(), + "time_now": utc_now(), "content": email_verification_content, "service_id": service_id, "subject": "Confirm GOV.UK Notify registration", @@ -107,7 +108,7 @@ def upgrade(): "template_id": "4f46df42-f795-4cc4-83bb-65ca312f49cc", "template_name": "Notify invitation email", "template_type": "email", - "time_now": datetime.utcnow(), + "time_now": utc_now(), "content": invitation_content, "service_id": service_id, "subject": invitation_subject, @@ -120,7 +121,7 @@ def upgrade(): "template_id": "4f46df42-f795-4cc4-83bb-65ca312f49cc", "template_name": "Notify invitation email", "template_type": "email", - "time_now": datetime.utcnow(), + "time_now": utc_now(), "content": invitation_content, "service_id": service_id, "subject": invitation_subject, @@ -135,7 +136,7 @@ def upgrade(): "template_id": "36fb0730-6259-4da1-8a80-c8de22ad4246", "template_name": "Notify SMS verify code", "template_type": "sms", - "time_now": datetime.utcnow(), + "time_now": utc_now(), "content": sms_code_content, "service_id": service_id, "subject": None, @@ -149,7 +150,7 @@ def upgrade(): "template_id": "36fb0730-6259-4da1-8a80-c8de22ad4246", "template_name": "Notify SMS verify code", "template_type": "sms", - "time_now": datetime.utcnow(), + "time_now": utc_now(), "content": sms_code_content, "service_id": service_id, "subject": None, @@ -172,7 +173,7 @@ def upgrade(): "template_id": "474e9242-823b-4f99-813d-ed392e7f1201", "template_name": "Notify password reset email", "template_type": "email", - "time_now": datetime.utcnow(), + "time_now": utc_now(), "content": password_reset_content, "service_id": service_id, "subject": "Reset your GOV.UK Notify password", @@ -186,7 +187,7 @@ def upgrade(): "template_id": "474e9242-823b-4f99-813d-ed392e7f1201", "template_name": "Notify password reset email", "template_type": "email", - "time_now": datetime.utcnow(), + "time_now": utc_now(), "content": password_reset_content, "service_id": service_id, "subject": "Reset your GOV.UK Notify password", diff --git a/migrations/versions/0028_fix_reg_template_history.py b/migrations/versions/0028_fix_reg_template_history.py index 8bf11fe59..fcbffc51a 100644 --- a/migrations/versions/0028_fix_reg_template_history.py +++ b/migrations/versions/0028_fix_reg_template_history.py @@ -11,6 +11,8 @@ from datetime import datetime from sqlalchemy import text +from app.utils import utc_now + revision = "0028_fix_reg_template_history" down_revision = "0026_rename_notify_service" @@ -38,7 +40,7 @@ def upgrade(): "id": "ece42649-22a8-4d06-b87f-d52d5d3f0a27", "name": "Notify email verification code", "type": "email", - "time_now": datetime.utcnow(), + "time_now": utc_now(), "content": email_verification_content, "service_id": service_id, "subject": "Confirm GOV.UK Notify registration", diff --git a/migrations/versions/0082_add_golive_template.py b/migrations/versions/0082_add_golive_template.py index 55cdc3e04..97ff5b97e 100644 --- a/migrations/versions/0082_add_golive_template.py +++ b/migrations/versions/0082_add_golive_template.py @@ -14,6 +14,8 @@ from alembic import op from flask import current_app from sqlalchemy import text +from app.utils import utc_now + revision = "0082_add_go_live_template" down_revision = "0081_noti_status_as_enum" @@ -89,7 +91,7 @@ GOV.UK Notify team "template_id": template_id, "template_name": template_name, "template_type": "email", - "time_now": datetime.utcnow(), + "time_now": utc_now(), "content": template_content, "notify_service_id": current_app.config["NOTIFY_SERVICE_ID"], "subject": template_subject, diff --git a/migrations/versions/0117_international_sms_notify.py b/migrations/versions/0117_international_sms_notify.py index ebdbbddef..c33750af4 100644 --- a/migrations/versions/0117_international_sms_notify.py +++ b/migrations/versions/0117_international_sms_notify.py @@ -9,6 +9,8 @@ Create Date: 2017-08-29 14:09:41.042061 # revision identifiers, used by Alembic. from sqlalchemy import text +from app.utils import utc_now + revision = "0117_international_sms_notify" down_revision = "0115_add_inbound_numbers" @@ -22,7 +24,7 @@ NOTIFY_SERVICE_ID = "d6aa2c68-a2d9-4437-ab19-3ae8eb202553" def upgrade(): input_params = { "notify_service_id": NOTIFY_SERVICE_ID, - "datetime_now": datetime.utcnow(), + "datetime_now": utc_now(), } conn = op.get_bind() conn.execute( diff --git a/migrations/versions/0134_add_email_2fa_template_.py b/migrations/versions/0134_add_email_2fa_template_.py index 492281175..57809e1bc 100644 --- a/migrations/versions/0134_add_email_2fa_template_.py +++ b/migrations/versions/0134_add_email_2fa_template_.py @@ -12,6 +12,8 @@ from alembic import op from flask import current_app from sqlalchemy import text +from app.utils import utc_now + revision = "0134_add_email_2fa_template" down_revision = "0133_set_services_sms_prefix" @@ -44,7 +46,7 @@ def upgrade(): "template_id": template_id, "template_name": template_name, "template_type": "email", - "time_now": datetime.utcnow(), + "time_now": utc_now(), "content": template_content, "notify_service_id": current_app.config["NOTIFY_SERVICE_ID"], "subject": template_subject, diff --git a/migrations/versions/0139_migrate_sms_allowance_data.py b/migrations/versions/0139_migrate_sms_allowance_data.py index 8e7536bfb..0203f5563 100644 --- a/migrations/versions/0139_migrate_sms_allowance_data.py +++ b/migrations/versions/0139_migrate_sms_allowance_data.py @@ -13,6 +13,7 @@ from alembic import op from sqlalchemy import text from app.dao.date_util import get_current_calendar_year_start_year +from app.utils import utc_now revision = "0139_migrate_sms_allowance_data" down_revision = "0138_sms_sender_nullable" @@ -34,7 +35,7 @@ def upgrade(): input_params = { "current_year": current_year, "default_limit": default_limit, - "time_now": datetime.utcnow(), + "time_now": utc_now(), } insert_row_if_not_exist = """ INSERT INTO annual_billing diff --git a/migrations/versions/0171_add_org_invite_template.py b/migrations/versions/0171_add_org_invite_template.py index 5ec1925da..7c0b9df09 100644 --- a/migrations/versions/0171_add_org_invite_template.py +++ b/migrations/versions/0171_add_org_invite_template.py @@ -12,6 +12,8 @@ from alembic import op from flask import current_app from sqlalchemy import text +from app.utils import utc_now + revision = "0171_add_org_invite_template" down_revision = "0170_hidden_non_nullable" @@ -53,7 +55,7 @@ def upgrade(): "template_id": template_id, "template_name": template_name, "template_type": "email", - "time_now": datetime.utcnow(), + "time_now": utc_now(), "content": template_content, "notify_service_id": current_app.config["NOTIFY_SERVICE_ID"], "subject": template_subject, diff --git a/migrations/versions/0265_add_confirm_edit_templates.py b/migrations/versions/0265_add_confirm_edit_templates.py index 378891313..ad8d2470e 100644 --- a/migrations/versions/0265_add_confirm_edit_templates.py +++ b/migrations/versions/0265_add_confirm_edit_templates.py @@ -12,6 +12,8 @@ from alembic import op from flask import current_app from sqlalchemy import text +from app.utils import utc_now + revision = "0265_add_confirm_edit_templates" down_revision = "0264_add_folder_permissions_perm" @@ -57,7 +59,7 @@ def upgrade(): "template_id": email_template_id, "template_name": email_template_name, "template_type": "email", - "time_now": datetime.utcnow(), + "time_now": utc_now(), "content": email_template_content, "notify_service_id": current_app.config["NOTIFY_SERVICE_ID"], "subject": email_template_subject, @@ -78,7 +80,7 @@ def upgrade(): "template_id": mobile_template_id, "template_name": mobile_template_name, "template_type": "sms", - "time_now": datetime.utcnow(), + "time_now": utc_now(), "content": mobile_template_content, "notify_service_id": current_app.config["NOTIFY_SERVICE_ID"], "subject": None, diff --git a/migrations/versions/0294_add_verify_reply_to_.py b/migrations/versions/0294_add_verify_reply_to_.py index 305c52997..d37f75ea0 100644 --- a/migrations/versions/0294_add_verify_reply_to_.py +++ b/migrations/versions/0294_add_verify_reply_to_.py @@ -12,6 +12,8 @@ from alembic import op from flask import current_app from sqlalchemy import text +from app.utils import utc_now + revision = "0294_add_verify_reply_to" down_revision = "0293_drop_complaint_fk" @@ -58,7 +60,7 @@ def upgrade(): "template_id": email_template_id, "template_name": email_template_name, "template_type": "email", - "time_now": datetime.utcnow(), + "time_now": utc_now(), "content": email_template_content, "notify_service_id": current_app.config["NOTIFY_SERVICE_ID"], "subject": email_template_subject, diff --git a/migrations/versions/0330_broadcast_invite_email.py b/migrations/versions/0330_broadcast_invite_email.py index 24dc60c68..bcd865d46 100644 --- a/migrations/versions/0330_broadcast_invite_email.py +++ b/migrations/versions/0330_broadcast_invite_email.py @@ -12,6 +12,8 @@ from datetime import datetime from alembic import op from sqlalchemy import text +from app.utils import utc_now + revision = "0330_broadcast_invite_email" down_revision = "0329_purge_broadcast_data" @@ -60,7 +62,7 @@ def upgrade(): input_params = { "template_id": template_id, "template_name": broadcast_invitation_template_name, - "time_now": datetime.utcnow(), + "time_now": utc_now(), "content": broadcast_invitation_content, "service_id": service_id, "subject": broadcast_invitation_subject, diff --git a/migrations/versions/0347_add_dvla_volumes_template.py b/migrations/versions/0347_add_dvla_volumes_template.py index 6b610c16c..f32821b79 100644 --- a/migrations/versions/0347_add_dvla_volumes_template.py +++ b/migrations/versions/0347_add_dvla_volumes_template.py @@ -13,6 +13,8 @@ from alembic import op from flask import current_app from sqlalchemy import text +from app.utils import utc_now + revision = "0347_add_dvla_volumes_template" down_revision = "0346_notify_number_sms_sender" @@ -57,7 +59,7 @@ def upgrade(): "template_id": email_template_id, "template_name": email_template_name, "template_type": "email", - "time_now": datetime.utcnow(), + "time_now": utc_now(), "content": email_template_content, "notify_service_id": current_app.config["NOTIFY_SERVICE_ID"], "subject": email_template_subject, diff --git a/migrations/versions/0401_add_e2e_test_user.py b/migrations/versions/0401_add_e2e_test_user.py index d3d83afad..e99a6af8a 100644 --- a/migrations/versions/0401_add_e2e_test_user.py +++ b/migrations/versions/0401_add_e2e_test_user.py @@ -16,6 +16,7 @@ from alembic import op from app import db from app.dao.users_dao import get_user_by_email from app.models import User +from app.utils import utc_now revision = "0401_add_e2e_test_user" down_revision = "0400_add_total_message_limit" @@ -32,11 +33,11 @@ def upgrade(): "password": password, "mobile_number": "+12025555555", "state": "active", - "created_at": datetime.datetime.utcnow(), - "password_changed_at": datetime.datetime.utcnow(), + "created_at": utc_now(), + "password_changed_at": utc_now(), "failed_login_count": 0, "platform_admin": "f", - "email_access_validated_at": datetime.datetime.utcnow(), + "email_access_validated_at": utc_now(), } conn = op.get_bind() insert_sql = """ diff --git a/tests/app/dao/test_fact_notification_status_dao.py b/tests/app/dao/test_fact_notification_status_dao.py index 4c7030b2e..dc46de45d 100644 --- a/tests/app/dao/test_fact_notification_status_dao.py +++ b/tests/app/dao/test_fact_notification_status_dao.py @@ -33,31 +33,44 @@ def test_fetch_notification_status_for_service_by_month(notify_db_session): service_1 = create_service(service_name="service_1") service_2 = create_service(service_name="service_2") - create_ft_notification_status( - date(2018, 1, 1), NotificationType.SMS, service_1, count=4 - ) - create_ft_notification_status( - date(2018, 1, 2), NotificationType.SMS, service_1, count=10 - ) - create_ft_notification_status( - date(2018, 1, 2), - NotificationType.SMS, - service_1, - notification_status=NotificationStatus.CREATED, - ) - create_ft_notification_status(date(2018, 1, 3), NotificationType.EMAIL, service_1) + create_template(service=service_1) + create_template(service=service_1, template_type=TemplateType.EMAIL) + # not the service being tested + create_template(service=service_2) - create_ft_notification_status(date(2018, 2, 2), NotificationType.SMS, service_1) + # loop messages for the month + for x in range(0, 14): + create_notification( + service_1.templates[0], + created_at=datetime(2018, 1, 1, 1, x, 0), + status=NotificationStatus.DELIVERED, + ) + create_notification( + service_1.templates[0], created_at=datetime(2018, 1, 1, 1, 1, 0) + ) + create_notification( + service_1.templates[1], + created_at=datetime(2018, 1, 1, 1, 1, 0), + status=NotificationStatus.DELIVERED, + ) + create_notification( + service_1.templates[0], + created_at=datetime(2018, 2, 1, 1, 1, 0), + status=NotificationStatus.DELIVERED, + ) - # not included - too early - create_ft_notification_status(date(2017, 12, 31), NotificationType.SMS, service_1) - # not included - too late - create_ft_notification_status(date(2017, 3, 1), NotificationType.SMS, service_1) - # not included - wrong service - create_ft_notification_status(date(2018, 1, 3), NotificationType.SMS, service_2) - # not included - test keys - create_ft_notification_status( - date(2018, 1, 3), NotificationType.SMS, service_1, key_type=KeyType.TEST + # not the right month + create_notification( + service_1.templates[0], + created_at=datetime(2018, 4, 1, 1, 1, 0), + status=NotificationStatus.DELIVERED, + ) + + # not the right service + create_notification( + service_2.templates[0], + created_at=datetime(2018, 2, 1, 1, 1, 0), + status=NotificationStatus.DELIVERED, ) results = sorted( diff --git a/tests/app/service/test_statistics.py b/tests/app/service/test_statistics.py index 28484a8d6..c760d01b8 100644 --- a/tests/app/service/test_statistics.py +++ b/tests/app/service/test_statistics.py @@ -301,12 +301,15 @@ def test_add_monthly_notification_status_stats(): data = create_empty_monthly_notification_status_stats_dict(2018) # this data won't be affected data["2018-05"][NotificationType.EMAIL][NotificationStatus.SENDING] = 32 + data["2018-05"][NotificationType.EMAIL][StatisticsType.REQUESTED] = 32 # this data will get combined with the 8 from row_data data["2018-05"][NotificationType.SMS][NotificationStatus.SENDING] = 16 + data["2018-05"][NotificationType.SMS][StatisticsType.REQUESTED] = 16 add_monthly_notification_status_stats(data, rows) # first 3 months are empty + assert data == { "2018-01": {NotificationType.SMS: {}, NotificationType.EMAIL: {}}, "2018-02": {NotificationType.SMS: {}, NotificationType.EMAIL: {}}, @@ -315,12 +318,22 @@ def test_add_monthly_notification_status_stats(): NotificationType.SMS: { NotificationStatus.SENDING: 1, NotificationStatus.DELIVERED: 2, + StatisticsType.REQUESTED: 3, + }, + NotificationType.EMAIL: { + NotificationStatus.SENDING: 4, + StatisticsType.REQUESTED: 4, }, - NotificationType.EMAIL: {NotificationStatus.SENDING: 4}, }, "2018-05": { - NotificationType.SMS: {NotificationStatus.SENDING: 24}, - NotificationType.EMAIL: {NotificationStatus.SENDING: 32}, + NotificationType.SMS: { + NotificationStatus.SENDING: 24, + StatisticsType.REQUESTED: 24, + }, + NotificationType.EMAIL: { + NotificationStatus.SENDING: 32, + StatisticsType.REQUESTED: 32, + }, }, "2018-06": {NotificationType.SMS: {}, NotificationType.EMAIL: {}}, } diff --git a/tests/app/service/test_statistics_rest.py b/tests/app/service/test_statistics_rest.py index 591e5ca8e..6d20cacc3 100644 --- a/tests/app/service/test_statistics_rest.py +++ b/tests/app/service/test_statistics_rest.py @@ -234,17 +234,36 @@ def test_get_monthly_notification_stats_returns_stats(admin_request, sample_serv sms_t2 = create_template(sample_service) email_template = create_template(sample_service, template_type=TemplateType.EMAIL) - create_ft_notification_status(datetime(2016, 6, 1), template=sms_t1) - create_ft_notification_status(datetime(2016, 6, 2), template=sms_t1) - - create_ft_notification_status(datetime(2016, 7, 1), template=sms_t1) - create_ft_notification_status(datetime(2016, 7, 1), template=sms_t2) - create_ft_notification_status( - datetime(2016, 7, 1), - template=sms_t1, - notification_status=NotificationStatus.CREATED, + create_notification( + sms_t1, + created_at=datetime(2016, 6, 1, 1, 1, 0), + status=NotificationStatus.DELIVERED, + ) + create_notification( + sms_t1, + created_at=datetime(2016, 6, 2, 1, 1, 0), + status=NotificationStatus.DELIVERED, + ) + create_notification( + sms_t1, + created_at=datetime(2016, 7, 1, 1, 1, 0), + status=NotificationStatus.DELIVERED, + ) + create_notification( + sms_t2, + created_at=datetime(2016, 7, 1, 1, 1, 0), + status=NotificationStatus.DELIVERED, + ) + create_notification( + sms_t1, + created_at=datetime(2016, 7, 1, 1, 1, 0), + status=NotificationStatus.CREATED, + ) + create_notification( + email_template, + created_at=datetime(2016, 7, 1, 1, 1, 0), + status=NotificationStatus.DELIVERED, ) - create_ft_notification_status(datetime(2016, 7, 1), template=email_template) response = admin_request.get( "service.get_monthly_notification_stats", @@ -256,7 +275,8 @@ def test_get_monthly_notification_stats_returns_stats(admin_request, sample_serv assert response["data"]["2016-06"] == { NotificationType.SMS: { # it combines the two days - NotificationStatus.DELIVERED: 2 + NotificationStatus.DELIVERED: 2, + StatisticsType.REQUESTED: 2, }, NotificationType.EMAIL: {}, } @@ -265,86 +285,43 @@ def test_get_monthly_notification_stats_returns_stats(admin_request, sample_serv NotificationType.SMS: { NotificationStatus.CREATED: 1, NotificationStatus.DELIVERED: 2, + StatisticsType.REQUESTED: 3, }, - NotificationType.EMAIL: {StatisticsType.DELIVERED: 1}, - } - - -@freeze_time("2016-06-05 12:00:00") -def test_get_monthly_notification_stats_combines_todays_data_and_historic_stats( - admin_request, sample_template -): - create_ft_notification_status( - datetime(2016, 5, 1, 12), - template=sample_template, - count=1, - ) - create_ft_notification_status( - datetime(2016, 6, 1, 12), - template=sample_template, - notification_status=NotificationStatus.CREATED, - count=2, - ) # noqa - - create_notification( - sample_template, - created_at=datetime(2016, 6, 5, 12), - status=NotificationStatus.CREATED, - ) - create_notification( - sample_template, - created_at=datetime(2016, 6, 5, 12), - status=NotificationStatus.DELIVERED, - ) - - # this doesn't get returned in the stats because it is old - it should be in ft_notification_status by now - create_notification( - sample_template, - created_at=datetime(2016, 6, 4, 12), - status=NotificationStatus.SENDING, - ) - - response = admin_request.get( - "service.get_monthly_notification_stats", - service_id=sample_template.service_id, - year=2016, - ) - - assert len(response["data"]) == 6 # January to June - assert response["data"]["2016-05"] == { - NotificationType.SMS: {NotificationStatus.DELIVERED: 1}, - NotificationType.EMAIL: {}, - } - assert response["data"]["2016-06"] == { - NotificationType.SMS: { - # combines the stats from the historic ft_notification_status and the current notifications - NotificationStatus.CREATED: 3, - NotificationStatus.DELIVERED: 1, + NotificationType.EMAIL: { + StatisticsType.DELIVERED: 1, + StatisticsType.REQUESTED: 1, }, - NotificationType.EMAIL: {}, } def test_get_monthly_notification_stats_ignores_test_keys( admin_request, sample_service ): - create_ft_notification_status( - datetime(2016, 6, 1), - service=sample_service, + create_template(service=sample_service) + + create_notification( + sample_service.templates[0], + created_at=datetime(2016, 6, 1, 1, 1, 0), key_type=KeyType.NORMAL, - count=1, + status=NotificationStatus.DELIVERED, ) - create_ft_notification_status( - datetime(2016, 6, 1), - service=sample_service, + create_notification( + sample_service.templates[0], + created_at=datetime(2016, 6, 2, 1, 1, 0), + key_type=KeyType.NORMAL, + status=NotificationStatus.DELIVERED, + ) + create_notification( + sample_service.templates[0], + created_at=datetime(2016, 6, 1, 1, 1, 0), key_type=KeyType.TEAM, - count=2, + status=NotificationStatus.DELIVERED, ) - create_ft_notification_status( - datetime(2016, 6, 1), - service=sample_service, + create_notification( + sample_service.templates[0], + created_at=datetime(2016, 6, 1, 1, 1, 0), key_type=KeyType.TEST, - count=4, + status=NotificationStatus.DELIVERED, ) response = admin_request.get( @@ -355,26 +332,27 @@ def test_get_monthly_notification_stats_ignores_test_keys( assert response["data"]["2016-06"][NotificationType.SMS] == { NotificationStatus.DELIVERED: 3, + StatisticsType.REQUESTED: 3, } def test_get_monthly_notification_stats_checks_dates(admin_request, sample_service): t = create_template(sample_service) - # create_ft_notification_status(datetime(2016, 3, 31), template=t, notification_status='created') - create_ft_notification_status( - datetime(2016, 4, 2), - template=t, - notification_status=NotificationStatus.SENDING, + + create_notification( + t, + created_at=datetime(2016, 4, 2), + status=NotificationStatus.SENDING, ) - create_ft_notification_status( - datetime(2017, 3, 31), - template=t, - notification_status=NotificationStatus.DELIVERED, + create_notification( + t, + created_at=datetime(2017, 3, 31), + status=NotificationStatus.DELIVERED, ) - create_ft_notification_status( - datetime(2017, 4, 11), - template=t, - notification_status=NotificationStatus.PERMANENT_FAILURE, + create_notification( + t, + created_at=datetime(2017, 4, 11), + status=NotificationStatus.PERMANENT_FAILURE, ) response = admin_request.get( @@ -386,9 +364,11 @@ def test_get_monthly_notification_stats_checks_dates(admin_request, sample_servi assert "2017-04" not in response["data"] assert response["data"]["2016-04"][NotificationType.SMS] == { NotificationStatus.SENDING: 1, + StatisticsType.REQUESTED: 1, } assert response["data"]["2016-04"][NotificationType.SMS] == { NotificationStatus.SENDING: 1, + StatisticsType.REQUESTED: 1, } @@ -399,15 +379,15 @@ def test_get_monthly_notification_stats_only_gets_for_one_service( templates = [create_template(services[0]), create_template(services[1])] - create_ft_notification_status( - datetime(2016, 6, 1), - template=templates[0], - notification_status=NotificationStatus.CREATED, + create_notification( + templates[0], + created_at=datetime(2016, 6, 1), + status=NotificationStatus.CREATED, ) - create_ft_notification_status( - datetime(2016, 6, 1), - template=templates[1], - notification_status=NotificationStatus.DELIVERED, + create_notification( + templates[1], + created_at=datetime(2016, 6, 1), + status=NotificationStatus.DELIVERED, ) response = admin_request.get( @@ -417,6 +397,9 @@ def test_get_monthly_notification_stats_only_gets_for_one_service( ) assert response["data"]["2016-06"] == { - NotificationType.SMS: {NotificationStatus.CREATED: 1}, + NotificationType.SMS: { + NotificationStatus.CREATED: 1, + StatisticsType.REQUESTED: 1, + }, NotificationType.EMAIL: {}, }