From 7ef70c39b9d4af3d7ac9a2b5d9f832aa06f910a3 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Tue, 2 Apr 2024 11:43:01 -0700 Subject: [PATCH 01/45] show all users --- app/commands.py | 20 ++++++++++++++++++++ tests/app/test_commands.py | 9 +++++++++ 2 files changed, 29 insertions(+) diff --git a/app/commands.py b/app/commands.py index 725e7ee99..38d49f198 100644 --- a/app/commands.py +++ b/app/commands.py @@ -1006,3 +1006,23 @@ def add_test_users_to_db(generate, state, admin): platform_admin=admin, ) print(f"{num} {user.email_address} created") + + +@notify_command(name="show-users") +def show_users(): + + sql = """ + + select users.name, users.email_address, users.mobile_number, services.name as service_name + from users + inner join user_to_service on users.id=user_to_service.user_id + inner join services on services.id=user_to_service.service_id + order by services.name asc, users.name asc + """ + users = db.session.execute(sql) + report = "Name,Email address,Mobile number,Service name" + print(report) + for row in users: + print(f"{row.name},{row.email_address},{row.mobile_number},{row.service_name}") + report = f"{report}\n{row.name},{row.email_address},{row.mobile_number},{row.service_name}" + return report diff --git a/tests/app/test_commands.py b/tests/app/test_commands.py index a96eae599..0f52a904c 100644 --- a/tests/app/test_commands.py +++ b/tests/app/test_commands.py @@ -16,6 +16,7 @@ from app.commands import ( populate_organizations_from_file, promote_user_to_platform_admin, purge_functional_test_data, + show_users, update_jobs_archived_flag, ) from app.dao.inbound_numbers_dao import dao_get_available_inbound_numbers @@ -440,3 +441,11 @@ def test_promote_user_to_platform_admin_no_result_found( ) assert "NoResultFound" in str(result) assert sample_user.platform_admin is False + + +def test_show_users(notify_db_session, notify_api, sample_user): + result = notify_api.test_cli_runner().invoke( + show_users, + [], + ) + assert "Name,Email address,Mobile number,Service name" in str(result) From 5cd68e808143e341735d71e96a628ef69a10bdcc Mon Sep 17 00:00:00 2001 From: Anastasia Gradova Date: Wed, 22 May 2024 13:33:32 -0600 Subject: [PATCH 02/45] New endpoints for #1006 and #1007 --- app/dao/services_dao.py | 27 +++++++++++++++++++++++++++ app/service/rest.py | 33 ++++++++++++++++++++++++++++++++- 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index df8e59287..bc2a5ddf3 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -425,6 +425,33 @@ def dao_fetch_todays_stats_for_service(service_id): ) +def dao_fetch_stats_for_service_from_day(service_id, day): + # today = datetime.now(timezone.utc).date() + # 2024-05-20 + # day_date = datetime.strptime(day, '%Y-%m-%d').date() + start_date = get_midnight_in_utc(day) + end_date = get_midnight_in_utc(day + timedelta(days=1)) + print(start_date) + return ( + db.session.query( + NotificationHistory.notification_type, + NotificationHistory.status, + func.count(NotificationHistory.id).label("count"), + ) + .filter( + NotificationHistory.service_id == service_id, + NotificationHistory.key_type != KeyType.TEST, + NotificationHistory.created_at >= start_date, + NotificationHistory.created_at <= end_date, + ) + .group_by( + NotificationHistory.notification_type, + NotificationHistory.status, + ) + .all() + ) + + def dao_fetch_todays_stats_for_all_services( include_from_test_key=True, only_active=True ): diff --git a/app/service/rest.py b/app/service/rest.py index ce5083073..9c0295304 100644 --- a/app/service/rest.py +++ b/app/service/rest.py @@ -1,5 +1,5 @@ import itertools -from datetime import datetime +from datetime import datetime, timedelta from flask import Blueprint, current_app, jsonify, request from sqlalchemy.exc import IntegrityError @@ -63,6 +63,7 @@ from app.dao.services_dao import ( dao_fetch_all_services_by_user, dao_fetch_live_services_data, dao_fetch_service_by_id, + dao_fetch_stats_for_service_from_day, dao_fetch_todays_stats_for_all_services, dao_fetch_todays_stats_for_service, dao_remove_user_from_service, @@ -210,6 +211,36 @@ def get_service_notification_statistics(service_id): ) +@service_blueprint.route("//statistics//") +def get_service_notification_statistics_by_day(service_id, start, days): + return jsonify( + data=get_service_statistics_for_specific_days(service_id, start, int(days)) + ) + + +def get_service_statistics_for_specific_days(service_id, start, days=1): + start_date = datetime.strptime(start, "%Y-%m-%d").date() + + if days == 1: + stats = {} + stats[start] = { + "value": statistics.format_statistics( + dao_fetch_stats_for_service_from_day(service_id, start_date) + ) + } + else: + stats = {} + for d in range(days): + new_date = start_date + timedelta(days=d) + key = new_date.strftime("%Y-%m-%d") + value = statistics.format_statistics( + dao_fetch_stats_for_service_from_day(service_id, new_date) + ) + stats[key] = {"value": value} + + return stats + + @service_blueprint.route("", methods=["POST"]) def create_service(): data = request.get_json() From 5fbad5bd69cdd453900a97efc1a30c44d261fcf5 Mon Sep 17 00:00:00 2001 From: Anastasia Gradova Date: Sat, 25 May 2024 20:59:08 -0600 Subject: [PATCH 03/45] Added endpoints for #1006 and #1007 --- app/dao/date_util.py | 5 +++ app/dao/services_dao.py | 54 +++++++++++++++++++++++- app/service/rest.py | 86 +++++++++++++++++++++++++++++++++++---- app/service/statistics.py | 3 +- 4 files changed, 137 insertions(+), 11 deletions(-) diff --git a/app/dao/date_util.py b/app/dao/date_util.py index 7aafd711f..66aadc9df 100644 --- a/app/dao/date_util.py +++ b/app/dao/date_util.py @@ -1,3 +1,4 @@ +import calendar from datetime import date, datetime, time, timedelta @@ -64,3 +65,7 @@ def get_calendar_year_for_datetime(start_date): return year - 1 else: return year + + +def get_number_of_days_for_month(year, month): + return calendar.monthrange(year, month)[1] diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index bc2a5ddf3..f6724f247 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -431,7 +431,6 @@ def dao_fetch_stats_for_service_from_day(service_id, day): # day_date = datetime.strptime(day, '%Y-%m-%d').date() start_date = get_midnight_in_utc(day) end_date = get_midnight_in_utc(day + timedelta(days=1)) - print(start_date) return ( db.session.query( NotificationHistory.notification_type, @@ -452,6 +451,33 @@ def dao_fetch_stats_for_service_from_day(service_id, day): ) +def dao_fetch_stats_for_service_from_day_for_user(service_id, day, user_id): + # today = datetime.now(timezone.utc).date() + # 2024-05-20 + # day_date = datetime.strptime(day, '%Y-%m-%d').date() + start_date = get_midnight_in_utc(day) + end_date = get_midnight_in_utc(day + timedelta(days=1)) + return ( + db.session.query( + NotificationHistory.notification_type, + NotificationHistory.status, + func.count(NotificationHistory.id).label("count"), + ) + .filter( + NotificationHistory.service_id == service_id, + NotificationHistory.key_type != KeyType.TEST, + NotificationHistory.created_at >= start_date, + NotificationHistory.created_at <= end_date, + NotificationHistory.created_by_id == user_id, + ) + .group_by( + NotificationHistory.notification_type, + NotificationHistory.status, + ) + .all() + ) + + def dao_fetch_todays_stats_for_all_services( include_from_test_key=True, only_active=True ): @@ -633,3 +659,29 @@ def get_live_services_with_organization(): ) return query.all() + + +def fetch_notification_stats_for_service_by_month_by_user( + start_date, end_date, service_id, user_id +): + return ( + db.session.query( + func.date_trunc("month", NotificationHistory.created_at).label("month"), + NotificationHistory.notification_type, + (NotificationHistory.status).label("notification_status"), + func.count(NotificationHistory.id).label("count"), + ) + .filter( + NotificationHistory.service_id == service_id, + NotificationHistory.created_at >= start_date, + NotificationHistory.created_at < end_date, + NotificationHistory.key_type != KeyType.TEST, + NotificationHistory.created_by_id == user_id, + ) + .group_by( + func.date_trunc("month", NotificationHistory.created_at).label("month"), + NotificationHistory.notification_type, + NotificationHistory.status, + ) + .all() + ) diff --git a/app/service/rest.py b/app/service/rest.py index 9c0295304..7d8d53c8e 100644 --- a/app/service/rest.py +++ b/app/service/rest.py @@ -17,7 +17,11 @@ from app.dao.api_key_dao import ( save_model_api_key, ) from app.dao.dao_utils import dao_rollback, transaction -from app.dao.date_util import get_calendar_year +from app.dao.date_util import ( + get_calendar_year, + get_month_start_and_end_date_in_utc, + get_number_of_days_for_month, +) from app.dao.fact_notification_status_dao import ( fetch_monthly_template_usage_for_service, fetch_notification_status_for_service_by_month, @@ -64,12 +68,14 @@ from app.dao.services_dao import ( dao_fetch_live_services_data, dao_fetch_service_by_id, dao_fetch_stats_for_service_from_day, + dao_fetch_stats_for_service_from_day_for_user, dao_fetch_todays_stats_for_all_services, dao_fetch_todays_stats_for_service, dao_remove_user_from_service, dao_resume_service, dao_suspend_service, dao_update_service, + fetch_notification_stats_for_service_by_month_by_user, get_services_by_partial_name, ) from app.dao.templates_dao import dao_get_template_by_id @@ -223,20 +229,17 @@ def get_service_statistics_for_specific_days(service_id, start, days=1): if days == 1: stats = {} - stats[start] = { - "value": statistics.format_statistics( - dao_fetch_stats_for_service_from_day(service_id, start_date) - ) - } + stats[start] = statistics.format_statistics( + dao_fetch_stats_for_service_from_day(service_id, start_date) + ) else: stats = {} for d in range(days): new_date = start_date + timedelta(days=d) key = new_date.strftime("%Y-%m-%d") - value = statistics.format_statistics( + stats[key] = statistics.format_statistics( dao_fetch_stats_for_service_from_day(service_id, new_date) ) - stats[key] = {"value": value} return stats @@ -612,6 +615,7 @@ def get_monthly_notification_stats(service_id): stats = fetch_notification_status_for_service_by_month( start_date, end_date, service_id ) + statistics.add_monthly_notification_status_stats(data, stats) now = datetime.utcnow() @@ -624,6 +628,72 @@ def get_monthly_notification_stats(service_id): return jsonify(data=data) +@service_blueprint.route( + "//notifications//monthly", methods=["GET"] +) +def get_monthly_notification_stats_by_user(service_id, user_id): + # check service_id validity + dao_fetch_service_by_id(service_id) + # user = get_user_by_id(user_id=user_id) + + try: + year = int(request.args.get("year", "NaN")) + except ValueError: + raise InvalidRequest("Year must be a number", status_code=400) + + start_date, end_date = get_calendar_year(year) + + data = statistics.create_empty_monthly_notification_status_stats_dict(year) + + stats = fetch_notification_stats_for_service_by_month_by_user( + start_date, end_date, service_id, user_id + ) + + statistics.add_monthly_notification_status_stats(data, stats) + + now = datetime.utcnow() + if end_date > now: + todays_deltas = fetch_notification_status_for_service_for_day( + now, service_id=service_id + ) + statistics.add_monthly_notification_status_stats(data, todays_deltas) + + return jsonify(data=data) + + +@service_blueprint.route( + "//notifications//month", methods=["GET"] +) +def get_single_month_notification_stats_by_user(service_id, user_id): + # check service_id validity + dao_fetch_service_by_id(service_id) + + try: + month = int(request.args.get("month", "NaN")) + year = int(request.args.get("year", "NaN")) + except ValueError: + raise InvalidRequest( + "Both a month and year are required as numbers", status_code=400 + ) + + month_year = datetime(year, month, 10, 00, 00, 00) + days = get_number_of_days_for_month(year, month) + start_date, end_date = get_month_start_and_end_date_in_utc(month_year) + + stats = {} + for d in range(days): + new_date = start_date + timedelta(days=d) + if new_date <= end_date: + key = new_date.strftime("%Y-%m-%d") + stats[key] = statistics.format_statistics( + dao_fetch_stats_for_service_from_day_for_user( + service_id, new_date, user_id + ) + ) + + return jsonify(stats) + + def get_detailed_service(service_id, today_only=False): service = dao_fetch_service_by_id(service_id) diff --git a/app/service/statistics.py b/app/service/statistics.py index 90b933960..a6b58e067 100644 --- a/app/service/statistics.py +++ b/app/service/statistics.py @@ -113,7 +113,6 @@ def create_empty_monthly_notification_status_stats_dict(year): def add_monthly_notification_status_stats(data, stats): for row in stats: month = row.month.strftime("%Y-%m") - data[month][row.notification_type][row.notification_status] += row.count - + data[month][row.notification_type][StatisticsType.REQUESTED] += row.count return data From 140e40ebe0b3d0cb4f052e6365f785f4b8f0e06a Mon Sep 17 00:00:00 2001 From: Anastasia Gradova Date: Sat, 25 May 2024 22:14:51 -0600 Subject: [PATCH 04/45] Updated pytest for new return values --- tests/app/service/test_statistics.py | 17 ++++++++++++++--- tests/app/service/test_statistics_rest.py | 23 +++++++++++++++++++---- 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/tests/app/service/test_statistics.py b/tests/app/service/test_statistics.py index 28484a8d6..b58b85f78 100644 --- a/tests/app/service/test_statistics.py +++ b/tests/app/service/test_statistics.py @@ -307,6 +307,7 @@ def test_add_monthly_notification_status_stats(): add_monthly_notification_status_stats(data, rows) # first 3 months are empty + print("********* ", data) assert data == { "2018-01": {NotificationType.SMS: {}, NotificationType.EMAIL: {}}, "2018-02": {NotificationType.SMS: {}, NotificationType.EMAIL: {}}, @@ -315,12 +316,22 @@ def test_add_monthly_notification_status_stats(): NotificationType.SMS: { NotificationStatus.SENDING: 1, NotificationStatus.DELIVERED: 2, + StatisticsType.REQUESTED: 3, + }, + NotificationType.EMAIL: { + NotificationStatus.SENDING: 4, + StatisticsType.REQUESTED: 4, }, - NotificationType.EMAIL: {NotificationStatus.SENDING: 4}, }, "2018-05": { - NotificationType.SMS: {NotificationStatus.SENDING: 24}, - NotificationType.EMAIL: {NotificationStatus.SENDING: 32}, + NotificationType.SMS: { + NotificationStatus.SENDING: 24, + StatisticsType.REQUESTED: 8, + }, + NotificationType.EMAIL: { + NotificationStatus.SENDING: 32, + StatisticsType.REQUESTED: 32, + }, }, "2018-06": {NotificationType.SMS: {}, NotificationType.EMAIL: {}}, } diff --git a/tests/app/service/test_statistics_rest.py b/tests/app/service/test_statistics_rest.py index 522c3902b..735730d63 100644 --- a/tests/app/service/test_statistics_rest.py +++ b/tests/app/service/test_statistics_rest.py @@ -255,7 +255,8 @@ def test_get_monthly_notification_stats_returns_stats(admin_request, sample_serv assert response["data"]["2016-06"] == { NotificationType.SMS: { # it combines the two days - NotificationStatus.DELIVERED: 2 + NotificationStatus.DELIVERED: 2, + StatisticsType.REQUESTED: 2, }, NotificationType.EMAIL: {}, } @@ -264,8 +265,12 @@ def test_get_monthly_notification_stats_returns_stats(admin_request, sample_serv NotificationType.SMS: { NotificationStatus.CREATED: 1, NotificationStatus.DELIVERED: 2, + StatisticsType.REQUESTED: 3, + }, + NotificationType.EMAIL: { + StatisticsType.DELIVERED: 1, + StatisticsType.REQUESTED: 1, }, - NotificationType.EMAIL: {StatisticsType.DELIVERED: 1}, } @@ -311,7 +316,10 @@ def test_get_monthly_notification_stats_combines_todays_data_and_historic_stats( assert len(response["data"]) == 6 # January to June assert response["data"]["2016-05"] == { - NotificationType.SMS: {NotificationStatus.DELIVERED: 1}, + NotificationType.SMS: { + NotificationStatus.DELIVERED: 1, + StatisticsType.REQUESTED: 1, + }, NotificationType.EMAIL: {}, } assert response["data"]["2016-06"] == { @@ -319,6 +327,7 @@ def test_get_monthly_notification_stats_combines_todays_data_and_historic_stats( # combines the stats from the historic ft_notification_status and the current notifications NotificationStatus.CREATED: 3, NotificationStatus.DELIVERED: 1, + StatisticsType.REQUESTED: 4, }, NotificationType.EMAIL: {}, } @@ -354,6 +363,7 @@ def test_get_monthly_notification_stats_ignores_test_keys( assert response["data"]["2016-06"][NotificationType.SMS] == { NotificationStatus.DELIVERED: 3, + StatisticsType.REQUESTED: 3, } @@ -385,9 +395,11 @@ def test_get_monthly_notification_stats_checks_dates(admin_request, sample_servi assert "2017-04" not in response["data"] assert response["data"]["2016-04"][NotificationType.SMS] == { NotificationStatus.SENDING: 1, + StatisticsType.REQUESTED: 1, } assert response["data"]["2016-04"][NotificationType.SMS] == { NotificationStatus.SENDING: 1, + StatisticsType.REQUESTED: 1, } @@ -416,6 +428,9 @@ def test_get_monthly_notification_stats_only_gets_for_one_service( ) assert response["data"]["2016-06"] == { - NotificationType.SMS: {NotificationStatus.CREATED: 1}, + NotificationType.SMS: { + NotificationStatus.CREATED: 1, + StatisticsType.REQUESTED: 1, + }, NotificationType.EMAIL: {}, } From 41f24162162fe9e138a6e41ee81279aa083ea946 Mon Sep 17 00:00:00 2001 From: Anastasia Gradova Date: Mon, 3 Jun 2024 21:38:29 -0600 Subject: [PATCH 05/45] reversed direction of the day count --- app/service/rest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/service/rest.py b/app/service/rest.py index 7d8d53c8e..0d015c08a 100644 --- a/app/service/rest.py +++ b/app/service/rest.py @@ -235,7 +235,7 @@ def get_service_statistics_for_specific_days(service_id, start, days=1): else: stats = {} for d in range(days): - new_date = start_date + timedelta(days=d) + new_date = start_date - timedelta(days=d) key = new_date.strftime("%Y-%m-%d") stats[key] = statistics.format_statistics( dao_fetch_stats_for_service_from_day(service_id, new_date) From a5055a0cf991aa710999b17258746839f7de9421 Mon Sep 17 00:00:00 2001 From: Anastasia Gradova Date: Thu, 6 Jun 2024 16:00:12 -0600 Subject: [PATCH 06/45] Updated endpoints to use the NotificationAllTimeView which is a view created to merge notifiations and notification_history --- app/dao/services_dao.py | 63 +++++++++++++++++++++-------------------- 1 file changed, 32 insertions(+), 31 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 36427d8a8..67fa69da2 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -28,6 +28,7 @@ from app.models import ( Job, Notification, NotificationHistory, + NotificationAllTimeView, Organization, Permission, Service, @@ -434,19 +435,19 @@ def dao_fetch_stats_for_service_from_day(service_id, day): end_date = get_midnight_in_utc(day + timedelta(days=1)) return ( db.session.query( - NotificationHistory.notification_type, - NotificationHistory.status, - func.count(NotificationHistory.id).label("count"), + NotificationAllTimeView.notification_type, + NotificationAllTimeView.status, + func.count(NotificationAllTimeView.id).label("count"), ) .filter( - NotificationHistory.service_id == service_id, - NotificationHistory.key_type != KeyType.TEST, - NotificationHistory.created_at >= start_date, - NotificationHistory.created_at <= end_date, + NotificationAllTimeView.service_id == service_id, + NotificationAllTimeView.key_type != KeyType.TEST, + NotificationAllTimeView.created_at >= start_date, + NotificationAllTimeView.created_at <= end_date, ) .group_by( - NotificationHistory.notification_type, - NotificationHistory.status, + NotificationAllTimeView.notification_type, + NotificationAllTimeView.status, ) .all() ) @@ -460,20 +461,20 @@ def dao_fetch_stats_for_service_from_day_for_user(service_id, day, user_id): end_date = get_midnight_in_utc(day + timedelta(days=1)) return ( db.session.query( - NotificationHistory.notification_type, - NotificationHistory.status, - func.count(NotificationHistory.id).label("count"), + NotificationAllTimeView.notification_type, + NotificationAllTimeView.status, + func.count(NotificationAllTimeView.id).label("count"), ) .filter( - NotificationHistory.service_id == service_id, - NotificationHistory.key_type != KeyType.TEST, - NotificationHistory.created_at >= start_date, - NotificationHistory.created_at <= end_date, - NotificationHistory.created_by_id == user_id, + NotificationAllTimeView.service_id == service_id, + NotificationAllTimeView.key_type != KeyType.TEST, + NotificationAllTimeView.created_at >= start_date, + NotificationAllTimeView.created_at <= end_date, + NotificationAllTimeView.created_by_id == user_id, ) .group_by( - NotificationHistory.notification_type, - NotificationHistory.status, + NotificationAllTimeView.notification_type, + NotificationAllTimeView.status, ) .all() ) @@ -667,22 +668,22 @@ def fetch_notification_stats_for_service_by_month_by_user( ): return ( db.session.query( - func.date_trunc("month", NotificationHistory.created_at).label("month"), - NotificationHistory.notification_type, - (NotificationHistory.status).label("notification_status"), - func.count(NotificationHistory.id).label("count"), + func.date_trunc("month", NotificationAllTimeView.created_at).label("month"), + NotificationAllTimeView.notification_type, + (NotificationAllTimeView.status).label("notification_status"), + func.count(NotificationAllTimeView.id).label("count"), ) .filter( - NotificationHistory.service_id == service_id, - NotificationHistory.created_at >= start_date, - NotificationHistory.created_at < end_date, - NotificationHistory.key_type != KeyType.TEST, - NotificationHistory.created_by_id == user_id, + NotificationAllTimeView.service_id == service_id, + NotificationAllTimeView.created_at >= start_date, + NotificationAllTimeView.created_at < end_date, + NotificationAllTimeView.key_type != KeyType.TEST, + NotificationAllTimeView.created_by_id == user_id, ) .group_by( - func.date_trunc("month", NotificationHistory.created_at).label("month"), - NotificationHistory.notification_type, - NotificationHistory.status, + func.date_trunc("month", NotificationAllTimeView.created_at).label("month"), + NotificationAllTimeView.notification_type, + NotificationAllTimeView.status, ) .all() ) From cd188180ca6f289411fcecbc067c747e488e55f9 Mon Sep 17 00:00:00 2001 From: Anastasia Gradova Date: Thu, 6 Jun 2024 16:37:12 -0600 Subject: [PATCH 07/45] Added new endpoint for getting statistics for a service, for a user, by a number of days --- app/service/rest.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/app/service/rest.py b/app/service/rest.py index 117b414df..dafccbbe3 100644 --- a/app/service/rest.py +++ b/app/service/rest.py @@ -243,6 +243,32 @@ def get_service_statistics_for_specific_days(service_id, start, days=1): return stats +@service_blueprint.route("//statistics/user///") +def get_service_notification_statistics_by_day_by_user(service_id, user_id, start, days): + return jsonify( + data=get_service_statistics_for_specific_days_by_user(service_id, user_id, start, int(days)) + ) + + +def get_service_statistics_for_specific_days_by_user(service_id, user_id, start, days=1): + start_date = datetime.strptime(start, "%Y-%m-%d").date() + + if days == 1: + stats = {} + stats[start] = statistics.format_statistics( + dao_fetch_stats_for_service_from_day_for_user(service_id, start_date, user_id) + ) + else: + stats = {} + for d in range(days): + new_date = start_date - timedelta(days=d) + key = new_date.strftime("%Y-%m-%d") + stats[key] = statistics.format_statistics( + dao_fetch_stats_for_service_from_day_for_user(service_id, new_date, user_id) + ) + + return stats + @service_blueprint.route("", methods=["POST"]) def create_service(): From 0082ba3dd0527b43c73d7f7dab3f58958a2ed622 Mon Sep 17 00:00:00 2001 From: Anastasia Gradova Date: Thu, 6 Jun 2024 22:11:24 -0600 Subject: [PATCH 08/45] Updated sort order, endpoint defenition, and tests for REQUESTED attribute --- app/dao/services_dao.py | 2 +- app/service/rest.py | 25 +++++++++++++++++++------ tests/app/service/test_statistics.py | 6 ++++-- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 67fa69da2..e2082a1ba 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -27,8 +27,8 @@ from app.models import ( InvitedUser, Job, Notification, - NotificationHistory, NotificationAllTimeView, + NotificationHistory, Organization, Permission, Service, diff --git a/app/service/rest.py b/app/service/rest.py index dafccbbe3..372dec7a5 100644 --- a/app/service/rest.py +++ b/app/service/rest.py @@ -243,20 +243,31 @@ def get_service_statistics_for_specific_days(service_id, start, days=1): return stats -@service_blueprint.route("//statistics/user///") -def get_service_notification_statistics_by_day_by_user(service_id, user_id, start, days): + +@service_blueprint.route( + "//statistics/user///" +) +def get_service_notification_statistics_by_day_by_user( + service_id, user_id, start, days +): return jsonify( - data=get_service_statistics_for_specific_days_by_user(service_id, user_id, start, int(days)) + data=get_service_statistics_for_specific_days_by_user( + service_id, user_id, start, int(days) + ) ) -def get_service_statistics_for_specific_days_by_user(service_id, user_id, start, days=1): +def get_service_statistics_for_specific_days_by_user( + service_id, user_id, start, days=1 +): start_date = datetime.strptime(start, "%Y-%m-%d").date() if days == 1: stats = {} stats[start] = statistics.format_statistics( - dao_fetch_stats_for_service_from_day_for_user(service_id, start_date, user_id) + dao_fetch_stats_for_service_from_day_for_user( + service_id, start_date, user_id + ) ) else: stats = {} @@ -264,7 +275,9 @@ def get_service_statistics_for_specific_days_by_user(service_id, user_id, start, new_date = start_date - timedelta(days=d) key = new_date.strftime("%Y-%m-%d") stats[key] = statistics.format_statistics( - dao_fetch_stats_for_service_from_day_for_user(service_id, new_date, user_id) + dao_fetch_stats_for_service_from_day_for_user( + service_id, new_date, user_id + ) ) return stats diff --git a/tests/app/service/test_statistics.py b/tests/app/service/test_statistics.py index b58b85f78..c760d01b8 100644 --- a/tests/app/service/test_statistics.py +++ b/tests/app/service/test_statistics.py @@ -301,13 +301,15 @@ def test_add_monthly_notification_status_stats(): data = create_empty_monthly_notification_status_stats_dict(2018) # this data won't be affected data["2018-05"][NotificationType.EMAIL][NotificationStatus.SENDING] = 32 + data["2018-05"][NotificationType.EMAIL][StatisticsType.REQUESTED] = 32 # this data will get combined with the 8 from row_data data["2018-05"][NotificationType.SMS][NotificationStatus.SENDING] = 16 + data["2018-05"][NotificationType.SMS][StatisticsType.REQUESTED] = 16 add_monthly_notification_status_stats(data, rows) # first 3 months are empty - print("********* ", data) + assert data == { "2018-01": {NotificationType.SMS: {}, NotificationType.EMAIL: {}}, "2018-02": {NotificationType.SMS: {}, NotificationType.EMAIL: {}}, @@ -326,7 +328,7 @@ def test_add_monthly_notification_status_stats(): "2018-05": { NotificationType.SMS: { NotificationStatus.SENDING: 24, - StatisticsType.REQUESTED: 8, + StatisticsType.REQUESTED: 24, }, NotificationType.EMAIL: { NotificationStatus.SENDING: 32, From bff2df514fb93c38e12aa65e40cff1b35242b6c5 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Fri, 14 Jun 2024 09:32:58 -0700 Subject: [PATCH 09/45] change it to a task --- app/aws/s3.py | 40 ++++++++++++++++++++++ app/celery/tasks.py | 39 +++++++++++++++++++-- app/commands.py | 20 ----------- app/config.py | 5 +++ app/dao/users_dao.py | 14 +++++++- app/notifications/process_notifications.py | 6 +++- tests/app/test_commands.py | 9 ----- tests/app/test_utils.py | 5 ++- 8 files changed, 104 insertions(+), 34 deletions(-) diff --git a/app/aws/s3.py b/app/aws/s3.py index 9466e6cce..8435b4938 100644 --- a/app/aws/s3.py +++ b/app/aws/s3.py @@ -1,4 +1,5 @@ import re +import uuid import botocore from boto3 import Session @@ -7,6 +8,7 @@ from flask import current_app from app import redis_store from app.clients import AWS_CLIENT_CONFIG +from notifications_utils.s3 import s3upload as utils_s3upload FILE_LOCATION_STRUCTURE = "service-{}-notify/{}.csv" @@ -19,11 +21,31 @@ JOBS_CACHE_HITS = "JOBS_CACHE_HITS" JOBS_CACHE_MISSES = "JOBS_CACHE_MISSES" +def get_csv_location(service_id, upload_id): + return ( + current_app.config["CSV_UPLOAD_BUCKET"]["bucket"], + FILE_LOCATION_STRUCTURE.format(service_id, upload_id), + current_app.config["CSV_UPLOAD_BUCKET"]["access_key_id"], + current_app.config["CSV_UPLOAD_BUCKET"]["secret_access_key"], + current_app.config["CSV_UPLOAD_BUCKET"]["region"], + ) + + def get_s3_file(bucket_name, file_location, access_key, secret_key, region): s3_file = get_s3_object(bucket_name, file_location, access_key, secret_key, region) return s3_file.get()["Body"].read().decode("utf-8") +def get_file_from_s3(file_location): + return get_s3_file( + current_app.config["CSV_UPLOAD_BUCKET"]["bucket"], + file_location, + current_app.config["CSV_UPLOAD_BUCKET"]["access_key_id"], + current_app.config["CSV_UPLOAD_BUCKET"]["secret_access_key"], + current_app.config["CSV_UPLOAD_BUCKET"]["region"], + ) + + def get_s3_object(bucket_name, file_location, access_key, secret_key, region): session = Session( aws_access_key_id=access_key, @@ -253,3 +275,21 @@ def remove_csv_object(object_key): current_app.config["CSV_UPLOAD_BUCKET"]["region"], ) return obj.delete() + + +def s3upload(service_id, filedata, upload_id=None): + + if upload_id is None: + upload_id = str(uuid.uuid4()) + bucket_name, file_location, access_key, secret_key, region = get_csv_location( + service_id, upload_id + ) + utils_s3upload( + filedata=filedata["data"], + region=region, + bucket_name=bucket_name, + file_location=file_location, + access_key=access_key, + secret_key=secret_key, + ) + return upload_id diff --git a/app/celery/tasks.py b/app/celery/tasks.py index 1950712af..342f21e4c 100644 --- a/app/celery/tasks.py +++ b/app/celery/tasks.py @@ -1,4 +1,5 @@ import json +import os from flask import current_app from requests import HTTPError, RequestException, request @@ -18,6 +19,7 @@ from app.dao.service_email_reply_to_dao import dao_get_reply_to_by_id from app.dao.service_inbound_api_dao import get_service_inbound_api_for_service from app.dao.service_sms_sender_dao import dao_get_service_sms_senders_by_id from app.dao.templates_dao import dao_get_template_by_id +from app.dao.users_dao import dao_report_users from app.enums import JobStatus, KeyType, NotificationType from app.notifications.process_notifications import persist_notification from app.notifications.validators import check_service_over_total_message_limit @@ -189,7 +191,11 @@ def save_sms(self, service_id, notification_id, encrypted_notification, sender_i # Return False when trial mode services try sending notifications # to non-team and non-simulated recipients. if not service_allowed_to_send_to(notification["to"], service, KeyType.NORMAL): - current_app.logger.info(hilite(scrub(f"service not allowed to send to {notification['to']}, aborting"))) + current_app.logger.info( + hilite( + scrub(f"service not allowed to send to {notification['to']}, aborting") + ) + ) current_app.logger.debug( "SMS {} failed as restricted service".format(notification_id) ) @@ -220,7 +226,9 @@ def save_sms(self, service_id, notification_id, encrypted_notification, sender_i ) # Kick off sns process in provider_tasks.py - current_app.logger.info(hilite(scrub(f"Going to deliver sms for recipient: {notification['to']}"))) + current_app.logger.info( + hilite(scrub(f"Going to deliver sms for recipient: {notification['to']}")) + ) provider_tasks.deliver_sms.apply_async( [str(saved_notification.id)], queue=QueueNames.SEND_SMS ) @@ -470,3 +478,30 @@ def process_incomplete_job(job_id): process_row(row, template, job, job.service, sender_id=sender_id) job_complete(job, resumed=True) + + +@notify_celery.task(name="report-all-users") +def report_all_users(): + """ + This is to support the platform admin's ability to view all user data. + It runs once per night and is stored in + bucket/service-all-users-report-{env}-notify/all-users-report-{env}.csv + + When the front end is ready, it can just download from there. + """ + users = dao_report_users() + csv_text = "NAME,EMAIL_ADDRESS,MOBILE_NUMBER,SERVICE\n" + for user in users: + row = f"{user[0]},{user[1]},{user[2]},{user[3]}\n" + csv_text = f"{csv_text}{row}" + my_env = os.getenv("NOTIFY_ENVIRONMENT") + report_name = f"all-users-report-{my_env}" + file_data = {} + file_data["data"] = csv_text + object_key = s3.FILE_LOCATION_STRUCTURE.format(report_name, report_name) + s3.remove_csv_object(object_key) + s3.s3upload(report_name, file_data, report_name) + + # prove that it works + x = s3.get_file_from_s3(object_key) + print(f"!!!!!!!DOWNLOADED {x}") diff --git a/app/commands.py b/app/commands.py index 3246f31fc..826c2013b 100644 --- a/app/commands.py +++ b/app/commands.py @@ -1008,23 +1008,3 @@ def add_test_users_to_db(generate, state, admin): platform_admin=admin, ) print(f"{num} {user.email_address} created") - - -@notify_command(name="show-users") -def show_users(): - - sql = """ - - select users.name, users.email_address, users.mobile_number, services.name as service_name - from users - inner join user_to_service on users.id=user_to_service.user_id - inner join services on services.id=user_to_service.service_id - order by services.name asc, users.name asc - """ - users = db.session.execute(sql) - report = "Name,Email address,Mobile number,Service name" - print(report) - for row in users: - print(f"{row.name},{row.email_address},{row.mobile_number},{row.service_name}") - report = f"{report}\n{row.name},{row.email_address},{row.mobile_number},{row.service_name}" - return report diff --git a/app/config.py b/app/config.py index 8d913bdd8..637ece32f 100644 --- a/app/config.py +++ b/app/config.py @@ -199,6 +199,11 @@ class Config(object): "schedule": timedelta(minutes=66), "options": {"queue": QueueNames.PERIODIC}, }, + "report-all-users": { + "task": "report-all-users", + "schedule": timedelta(minutes=2), + "options": {"queue": QueueNames.PERIODIC}, + }, "check-job-status": { "task": "check-job-status", "schedule": crontab(), diff --git a/app/dao/users_dao.py b/app/dao/users_dao.py index d7291b35c..e541f4052 100644 --- a/app/dao/users_dao.py +++ b/app/dao/users_dao.py @@ -4,7 +4,7 @@ from secrets import randbelow import sqlalchemy from flask import current_app -from sqlalchemy import func +from sqlalchemy import func, text from sqlalchemy.orm import joinedload from app import db @@ -244,3 +244,15 @@ def user_can_be_archived(user): return False return True + + +def dao_report_users(): + sql = """ + select users.name, users.email_address, users.mobile_number, services.name as service_name + from users + inner join user_to_service on users.id=user_to_service.user_id + inner join services on services.id=user_to_service.service_id + where services.name not like '_archived%' + order by services.name asc, users.name asc + """ + return db.session.execute(text(sql)) diff --git a/app/notifications/process_notifications.py b/app/notifications/process_notifications.py index d899a8146..a91c27158 100644 --- a/app/notifications/process_notifications.py +++ b/app/notifications/process_notifications.py @@ -110,7 +110,11 @@ def persist_notification( formatted_recipient = validate_and_format_phone_number( recipient, international=True ) - current_app.logger.info(hilite(scrub(f"Persisting notification with recipient {formatted_recipient}"))) + current_app.logger.info( + hilite( + scrub(f"Persisting notification with recipient {formatted_recipient}") + ) + ) recipient_info = get_international_phone_info(formatted_recipient) notification.normalised_to = formatted_recipient notification.international = recipient_info.international diff --git a/tests/app/test_commands.py b/tests/app/test_commands.py index f1d5fd77a..7eee00bbf 100644 --- a/tests/app/test_commands.py +++ b/tests/app/test_commands.py @@ -16,7 +16,6 @@ from app.commands import ( populate_organizations_from_file, promote_user_to_platform_admin, purge_functional_test_data, - show_users, update_jobs_archived_flag, ) from app.dao.inbound_numbers_dao import dao_get_available_inbound_numbers @@ -442,11 +441,3 @@ def test_promote_user_to_platform_admin_no_result_found( ) assert "NoResultFound" in str(result) assert sample_user.platform_admin is False - - -def test_show_users(notify_db_session, notify_api, sample_user): - result = notify_api.test_cli_runner().invoke( - show_users, - [], - ) - assert "Name,Email address,Mobile number,Service name" in str(result) diff --git a/tests/app/test_utils.py b/tests/app/test_utils.py index bbe37256a..20675aec5 100644 --- a/tests/app/test_utils.py +++ b/tests/app/test_utils.py @@ -99,7 +99,10 @@ def test_scrub(): result = scrub( "This is a message with 17775554324, and also 18884449323 and also 17775554324" ) - assert result == "This is a message with 1XXXXX54324, and also 1XXXXX49323 and also 1XXXXX54324" + assert ( + result + == "This is a message with 1XXXXX54324, and also 1XXXXX49323 and also 1XXXXX54324" + ) # This method is used for simulating bulk sends. We use localstack and run on a developer's machine to do the From e293f7e3f5a26a13cc04d787bd0f3d7af65629b2 Mon Sep 17 00:00:00 2001 From: Anastasia Gradova Date: Fri, 14 Jun 2024 16:01:04 -0600 Subject: [PATCH 10/45] Updated all usage of datetime.utcnow() to app.utils utc_now() function. Added new endpoint /service/{{service_id}}/notifications/month --- app/service/rest.py | 31 ++++++++++++++++++- .../versions/0025_notify_service_data.py | 23 +++++++------- .../versions/0028_fix_reg_template_history.py | 4 ++- .../versions/0082_add_golive_template.py | 4 ++- .../versions/0117_international_sms_notify.py | 4 ++- .../versions/0134_add_email_2fa_template_.py | 4 ++- .../0139_migrate_sms_allowance_data.py | 3 +- .../versions/0171_add_org_invite_template.py | 4 ++- .../0265_add_confirm_edit_templates.py | 6 ++-- .../versions/0294_add_verify_reply_to_.py | 4 ++- .../versions/0330_broadcast_invite_email.py | 4 ++- .../0347_add_dvla_volumes_template.py | 4 ++- migrations/versions/0401_add_e2e_test_user.py | 7 +++-- 13 files changed, 76 insertions(+), 26 deletions(-) diff --git a/app/service/rest.py b/app/service/rest.py index 372dec7a5..29408f348 100644 --- a/app/service/rest.py +++ b/app/service/rest.py @@ -706,7 +706,7 @@ def get_monthly_notification_stats_by_user(service_id, user_id): statistics.add_monthly_notification_status_stats(data, stats) - now = datetime.utcnow() + now = utc_now() if end_date > now: todays_deltas = fetch_notification_status_for_service_for_day( now, service_id=service_id @@ -749,6 +749,35 @@ def get_single_month_notification_stats_by_user(service_id, user_id): return jsonify(stats) +@service_blueprint.route("//notifications/month", methods=["GET"]) +def get_single_month_notification_stats_for_service(service_id): + # check service_id validity + dao_fetch_service_by_id(service_id) + + try: + month = int(request.args.get("month", "NaN")) + year = int(request.args.get("year", "NaN")) + except ValueError: + raise InvalidRequest( + "Both a month and year are required as numbers", status_code=400 + ) + + month_year = datetime(year, month, 10, 00, 00, 00) + days = get_number_of_days_for_month(year, month) + start_date, end_date = get_month_start_and_end_date_in_utc(month_year) + + stats = {} + for d in range(days): + new_date = start_date + timedelta(days=d) + if new_date <= end_date: + key = new_date.strftime("%Y-%m-%d") + stats[key] = statistics.format_statistics( + dao_fetch_stats_for_service_from_day(service_id, new_date) + ) + + return jsonify(stats) + + def get_detailed_service(service_id, today_only=False): service = dao_fetch_service_by_id(service_id) diff --git a/migrations/versions/0025_notify_service_data.py b/migrations/versions/0025_notify_service_data.py index 0683e7dd2..e90d01aad 100644 --- a/migrations/versions/0025_notify_service_data.py +++ b/migrations/versions/0025_notify_service_data.py @@ -15,6 +15,7 @@ from alembic import op from sqlalchemy import text from app.hashing import hashpw +from app.utils import utc_now revision = "0025_notify_service_data" down_revision = "0024_add_research_mode_defaults" @@ -32,7 +33,7 @@ def upgrade(): """ conn.execute( text(user_insert), - {"user_id": user_id, "time_now": datetime.utcnow(), "password": password}, + {"user_id": user_id, "time_now": utc_now(), "password": password}, ) service_history_insert = """INSERT INTO services_history (id, name, created_at, active, message_limit, restricted, research_mode, email_from, created_by_id, reply_to_email_address, version) VALUES (:service_id, 'Notify service', :time_now, True, 1000, False, False, 'testsender@dispostable.com', @@ -41,7 +42,7 @@ def upgrade(): """ conn.execute( text(service_history_insert), - {"service_id": service_id, "time_now": datetime.utcnow(), "user_id": user_id}, + {"service_id": service_id, "time_now": utc_now(), "user_id": user_id}, ) service_insert = """INSERT INTO services (id, name, created_at, active, message_limit, restricted, research_mode, email_from, created_by_id, reply_to_email_address, version) VALUES (:service_id, 'Notify service', :time_now, True, 1000, False, False, 'testsender@dispostable.com', @@ -49,7 +50,7 @@ def upgrade(): """ conn.execute( text(service_insert), - {"service_id": service_id, "time_now": datetime.utcnow(), "user_id": user_id}, + {"service_id": service_id, "time_now": utc_now(), "user_id": user_id}, ) user_to_service_insert = """INSERT INTO user_to_service (user_id, service_id) VALUES (:user_id, :service_id)""" conn.execute( @@ -74,7 +75,7 @@ def upgrade(): "template_id": uuid.uuid4(), "template_name": "Notify email verification code", "template_type": "email", - "time_now": datetime.utcnow(), + "time_now": utc_now(), "content": email_verification_content, "service_id": service_id, "subject": "Confirm GOV.UK Notify registration", @@ -87,7 +88,7 @@ def upgrade(): "template_id": "ece42649-22a8-4d06-b87f-d52d5d3f0a27", "template_name": "Notify email verification code", "template_type": "email", - "time_now": datetime.utcnow(), + "time_now": utc_now(), "content": email_verification_content, "service_id": service_id, "subject": "Confirm GOV.UK Notify registration", @@ -107,7 +108,7 @@ def upgrade(): "template_id": "4f46df42-f795-4cc4-83bb-65ca312f49cc", "template_name": "Notify invitation email", "template_type": "email", - "time_now": datetime.utcnow(), + "time_now": utc_now(), "content": invitation_content, "service_id": service_id, "subject": invitation_subject, @@ -120,7 +121,7 @@ def upgrade(): "template_id": "4f46df42-f795-4cc4-83bb-65ca312f49cc", "template_name": "Notify invitation email", "template_type": "email", - "time_now": datetime.utcnow(), + "time_now": utc_now(), "content": invitation_content, "service_id": service_id, "subject": invitation_subject, @@ -135,7 +136,7 @@ def upgrade(): "template_id": "36fb0730-6259-4da1-8a80-c8de22ad4246", "template_name": "Notify SMS verify code", "template_type": "sms", - "time_now": datetime.utcnow(), + "time_now": utc_now(), "content": sms_code_content, "service_id": service_id, "subject": None, @@ -149,7 +150,7 @@ def upgrade(): "template_id": "36fb0730-6259-4da1-8a80-c8de22ad4246", "template_name": "Notify SMS verify code", "template_type": "sms", - "time_now": datetime.utcnow(), + "time_now": utc_now(), "content": sms_code_content, "service_id": service_id, "subject": None, @@ -172,7 +173,7 @@ def upgrade(): "template_id": "474e9242-823b-4f99-813d-ed392e7f1201", "template_name": "Notify password reset email", "template_type": "email", - "time_now": datetime.utcnow(), + "time_now": utc_now(), "content": password_reset_content, "service_id": service_id, "subject": "Reset your GOV.UK Notify password", @@ -186,7 +187,7 @@ def upgrade(): "template_id": "474e9242-823b-4f99-813d-ed392e7f1201", "template_name": "Notify password reset email", "template_type": "email", - "time_now": datetime.utcnow(), + "time_now": utc_now(), "content": password_reset_content, "service_id": service_id, "subject": "Reset your GOV.UK Notify password", diff --git a/migrations/versions/0028_fix_reg_template_history.py b/migrations/versions/0028_fix_reg_template_history.py index 8bf11fe59..fcbffc51a 100644 --- a/migrations/versions/0028_fix_reg_template_history.py +++ b/migrations/versions/0028_fix_reg_template_history.py @@ -11,6 +11,8 @@ from datetime import datetime from sqlalchemy import text +from app.utils import utc_now + revision = "0028_fix_reg_template_history" down_revision = "0026_rename_notify_service" @@ -38,7 +40,7 @@ def upgrade(): "id": "ece42649-22a8-4d06-b87f-d52d5d3f0a27", "name": "Notify email verification code", "type": "email", - "time_now": datetime.utcnow(), + "time_now": utc_now(), "content": email_verification_content, "service_id": service_id, "subject": "Confirm GOV.UK Notify registration", diff --git a/migrations/versions/0082_add_golive_template.py b/migrations/versions/0082_add_golive_template.py index 55cdc3e04..97ff5b97e 100644 --- a/migrations/versions/0082_add_golive_template.py +++ b/migrations/versions/0082_add_golive_template.py @@ -14,6 +14,8 @@ from alembic import op from flask import current_app from sqlalchemy import text +from app.utils import utc_now + revision = "0082_add_go_live_template" down_revision = "0081_noti_status_as_enum" @@ -89,7 +91,7 @@ GOV.UK Notify team "template_id": template_id, "template_name": template_name, "template_type": "email", - "time_now": datetime.utcnow(), + "time_now": utc_now(), "content": template_content, "notify_service_id": current_app.config["NOTIFY_SERVICE_ID"], "subject": template_subject, diff --git a/migrations/versions/0117_international_sms_notify.py b/migrations/versions/0117_international_sms_notify.py index ebdbbddef..c33750af4 100644 --- a/migrations/versions/0117_international_sms_notify.py +++ b/migrations/versions/0117_international_sms_notify.py @@ -9,6 +9,8 @@ Create Date: 2017-08-29 14:09:41.042061 # revision identifiers, used by Alembic. from sqlalchemy import text +from app.utils import utc_now + revision = "0117_international_sms_notify" down_revision = "0115_add_inbound_numbers" @@ -22,7 +24,7 @@ NOTIFY_SERVICE_ID = "d6aa2c68-a2d9-4437-ab19-3ae8eb202553" def upgrade(): input_params = { "notify_service_id": NOTIFY_SERVICE_ID, - "datetime_now": datetime.utcnow(), + "datetime_now": utc_now(), } conn = op.get_bind() conn.execute( diff --git a/migrations/versions/0134_add_email_2fa_template_.py b/migrations/versions/0134_add_email_2fa_template_.py index 492281175..57809e1bc 100644 --- a/migrations/versions/0134_add_email_2fa_template_.py +++ b/migrations/versions/0134_add_email_2fa_template_.py @@ -12,6 +12,8 @@ from alembic import op from flask import current_app from sqlalchemy import text +from app.utils import utc_now + revision = "0134_add_email_2fa_template" down_revision = "0133_set_services_sms_prefix" @@ -44,7 +46,7 @@ def upgrade(): "template_id": template_id, "template_name": template_name, "template_type": "email", - "time_now": datetime.utcnow(), + "time_now": utc_now(), "content": template_content, "notify_service_id": current_app.config["NOTIFY_SERVICE_ID"], "subject": template_subject, diff --git a/migrations/versions/0139_migrate_sms_allowance_data.py b/migrations/versions/0139_migrate_sms_allowance_data.py index 8e7536bfb..0203f5563 100644 --- a/migrations/versions/0139_migrate_sms_allowance_data.py +++ b/migrations/versions/0139_migrate_sms_allowance_data.py @@ -13,6 +13,7 @@ from alembic import op from sqlalchemy import text from app.dao.date_util import get_current_calendar_year_start_year +from app.utils import utc_now revision = "0139_migrate_sms_allowance_data" down_revision = "0138_sms_sender_nullable" @@ -34,7 +35,7 @@ def upgrade(): input_params = { "current_year": current_year, "default_limit": default_limit, - "time_now": datetime.utcnow(), + "time_now": utc_now(), } insert_row_if_not_exist = """ INSERT INTO annual_billing diff --git a/migrations/versions/0171_add_org_invite_template.py b/migrations/versions/0171_add_org_invite_template.py index 5ec1925da..7c0b9df09 100644 --- a/migrations/versions/0171_add_org_invite_template.py +++ b/migrations/versions/0171_add_org_invite_template.py @@ -12,6 +12,8 @@ from alembic import op from flask import current_app from sqlalchemy import text +from app.utils import utc_now + revision = "0171_add_org_invite_template" down_revision = "0170_hidden_non_nullable" @@ -53,7 +55,7 @@ def upgrade(): "template_id": template_id, "template_name": template_name, "template_type": "email", - "time_now": datetime.utcnow(), + "time_now": utc_now(), "content": template_content, "notify_service_id": current_app.config["NOTIFY_SERVICE_ID"], "subject": template_subject, diff --git a/migrations/versions/0265_add_confirm_edit_templates.py b/migrations/versions/0265_add_confirm_edit_templates.py index 378891313..ad8d2470e 100644 --- a/migrations/versions/0265_add_confirm_edit_templates.py +++ b/migrations/versions/0265_add_confirm_edit_templates.py @@ -12,6 +12,8 @@ from alembic import op from flask import current_app from sqlalchemy import text +from app.utils import utc_now + revision = "0265_add_confirm_edit_templates" down_revision = "0264_add_folder_permissions_perm" @@ -57,7 +59,7 @@ def upgrade(): "template_id": email_template_id, "template_name": email_template_name, "template_type": "email", - "time_now": datetime.utcnow(), + "time_now": utc_now(), "content": email_template_content, "notify_service_id": current_app.config["NOTIFY_SERVICE_ID"], "subject": email_template_subject, @@ -78,7 +80,7 @@ def upgrade(): "template_id": mobile_template_id, "template_name": mobile_template_name, "template_type": "sms", - "time_now": datetime.utcnow(), + "time_now": utc_now(), "content": mobile_template_content, "notify_service_id": current_app.config["NOTIFY_SERVICE_ID"], "subject": None, diff --git a/migrations/versions/0294_add_verify_reply_to_.py b/migrations/versions/0294_add_verify_reply_to_.py index 305c52997..d37f75ea0 100644 --- a/migrations/versions/0294_add_verify_reply_to_.py +++ b/migrations/versions/0294_add_verify_reply_to_.py @@ -12,6 +12,8 @@ from alembic import op from flask import current_app from sqlalchemy import text +from app.utils import utc_now + revision = "0294_add_verify_reply_to" down_revision = "0293_drop_complaint_fk" @@ -58,7 +60,7 @@ def upgrade(): "template_id": email_template_id, "template_name": email_template_name, "template_type": "email", - "time_now": datetime.utcnow(), + "time_now": utc_now(), "content": email_template_content, "notify_service_id": current_app.config["NOTIFY_SERVICE_ID"], "subject": email_template_subject, diff --git a/migrations/versions/0330_broadcast_invite_email.py b/migrations/versions/0330_broadcast_invite_email.py index 24dc60c68..bcd865d46 100644 --- a/migrations/versions/0330_broadcast_invite_email.py +++ b/migrations/versions/0330_broadcast_invite_email.py @@ -12,6 +12,8 @@ from datetime import datetime from alembic import op from sqlalchemy import text +from app.utils import utc_now + revision = "0330_broadcast_invite_email" down_revision = "0329_purge_broadcast_data" @@ -60,7 +62,7 @@ def upgrade(): input_params = { "template_id": template_id, "template_name": broadcast_invitation_template_name, - "time_now": datetime.utcnow(), + "time_now": utc_now(), "content": broadcast_invitation_content, "service_id": service_id, "subject": broadcast_invitation_subject, diff --git a/migrations/versions/0347_add_dvla_volumes_template.py b/migrations/versions/0347_add_dvla_volumes_template.py index 6b610c16c..f32821b79 100644 --- a/migrations/versions/0347_add_dvla_volumes_template.py +++ b/migrations/versions/0347_add_dvla_volumes_template.py @@ -13,6 +13,8 @@ from alembic import op from flask import current_app from sqlalchemy import text +from app.utils import utc_now + revision = "0347_add_dvla_volumes_template" down_revision = "0346_notify_number_sms_sender" @@ -57,7 +59,7 @@ def upgrade(): "template_id": email_template_id, "template_name": email_template_name, "template_type": "email", - "time_now": datetime.utcnow(), + "time_now": utc_now(), "content": email_template_content, "notify_service_id": current_app.config["NOTIFY_SERVICE_ID"], "subject": email_template_subject, diff --git a/migrations/versions/0401_add_e2e_test_user.py b/migrations/versions/0401_add_e2e_test_user.py index d3d83afad..e99a6af8a 100644 --- a/migrations/versions/0401_add_e2e_test_user.py +++ b/migrations/versions/0401_add_e2e_test_user.py @@ -16,6 +16,7 @@ from alembic import op from app import db from app.dao.users_dao import get_user_by_email from app.models import User +from app.utils import utc_now revision = "0401_add_e2e_test_user" down_revision = "0400_add_total_message_limit" @@ -32,11 +33,11 @@ def upgrade(): "password": password, "mobile_number": "+12025555555", "state": "active", - "created_at": datetime.datetime.utcnow(), - "password_changed_at": datetime.datetime.utcnow(), + "created_at": utc_now(), + "password_changed_at": utc_now(), "failed_login_count": 0, "platform_admin": "f", - "email_access_validated_at": datetime.datetime.utcnow(), + "email_access_validated_at": utc_now(), } conn = op.get_bind() insert_sql = """ From fd37923294a4103af28d851890fb75228e6a6ed3 Mon Sep 17 00:00:00 2001 From: Anastasia Gradova Date: Thu, 20 Jun 2024 01:04:31 -0600 Subject: [PATCH 11/45] Updated SQLAlchemy queries and API endpoints for single database queries to improve application performance. --- app/dao/services_dao.py | 24 +++--- app/service/rest.py | 167 +++++++++++++++++++++++++++------------- 2 files changed, 125 insertions(+), 66 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index e2082a1ba..7b16aa8b3 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -427,16 +427,15 @@ def dao_fetch_todays_stats_for_service(service_id): ) -def dao_fetch_stats_for_service_from_day(service_id, day): - # today = datetime.now(timezone.utc).date() - # 2024-05-20 - # day_date = datetime.strptime(day, '%Y-%m-%d').date() - start_date = get_midnight_in_utc(day) - end_date = get_midnight_in_utc(day + timedelta(days=1)) +def dao_fetch_stats_for_service_from_days(service_id, start, days): + start_date = get_midnight_in_utc(start) + end_date = get_midnight_in_utc(start + timedelta(days=days)) + return ( db.session.query( NotificationAllTimeView.notification_type, NotificationAllTimeView.status, + func.date_trunc("day", NotificationAllTimeView.created_at).label("day"), func.count(NotificationAllTimeView.id).label("count"), ) .filter( @@ -448,21 +447,21 @@ def dao_fetch_stats_for_service_from_day(service_id, day): .group_by( NotificationAllTimeView.notification_type, NotificationAllTimeView.status, + func.date_trunc("day", NotificationAllTimeView.created_at), ) .all() ) -def dao_fetch_stats_for_service_from_day_for_user(service_id, day, user_id): - # today = datetime.now(timezone.utc).date() - # 2024-05-20 - # day_date = datetime.strptime(day, '%Y-%m-%d').date() - start_date = get_midnight_in_utc(day) - end_date = get_midnight_in_utc(day + timedelta(days=1)) +def dao_fetch_stats_for_service_from_days_for_user(service_id, start, days, user_id): + start_date = get_midnight_in_utc(start) + end_date = get_midnight_in_utc(start + timedelta(days=days)) + return ( db.session.query( NotificationAllTimeView.notification_type, NotificationAllTimeView.status, + func.date_trunc("day", NotificationAllTimeView.created_at).label("day"), func.count(NotificationAllTimeView.id).label("count"), ) .filter( @@ -475,6 +474,7 @@ def dao_fetch_stats_for_service_from_day_for_user(service_id, day, user_id): .group_by( NotificationAllTimeView.notification_type, NotificationAllTimeView.status, + func.date_trunc("day", NotificationAllTimeView.created_at), ) .all() ) diff --git a/app/service/rest.py b/app/service/rest.py index 29408f348..10cd84b46 100644 --- a/app/service/rest.py +++ b/app/service/rest.py @@ -1,4 +1,5 @@ import itertools +from collections import defaultdict from datetime import datetime, timedelta from flask import Blueprint, current_app, jsonify, request @@ -17,11 +18,7 @@ from app.dao.api_key_dao import ( save_model_api_key, ) from app.dao.dao_utils import dao_rollback, transaction -from app.dao.date_util import ( - get_calendar_year, - get_month_start_and_end_date_in_utc, - get_number_of_days_for_month, -) +from app.dao.date_util import get_calendar_year, get_month_start_and_end_date_in_utc from app.dao.fact_notification_status_dao import ( fetch_monthly_template_usage_for_service, fetch_notification_status_for_service_by_month, @@ -67,8 +64,8 @@ from app.dao.services_dao import ( dao_fetch_all_services_by_user, dao_fetch_live_services_data, dao_fetch_service_by_id, - dao_fetch_stats_for_service_from_day, - dao_fetch_stats_for_service_from_day_for_user, + dao_fetch_stats_for_service_from_days, + dao_fetch_stats_for_service_from_days_for_user, dao_fetch_todays_stats_for_all_services, dao_fetch_todays_stats_for_service, dao_remove_user_from_service, @@ -227,19 +224,33 @@ def get_service_notification_statistics_by_day(service_id, start, days): def get_service_statistics_for_specific_days(service_id, start, days=1): start_date = datetime.strptime(start, "%Y-%m-%d").date() - if days == 1: - stats = {} - stats[start] = statistics.format_statistics( - dao_fetch_stats_for_service_from_day(service_id, start_date) - ) - else: - stats = {} - for d in range(days): - new_date = start_date - timedelta(days=d) - key = new_date.strftime("%Y-%m-%d") - stats[key] = statistics.format_statistics( - dao_fetch_stats_for_service_from_day(service_id, new_date) - ) + def generate_date_range(start_date, days): + current_date = start_date + end_date = start_date - timedelta(days=days) + while current_date > end_date: + try: + valid_date = datetime( + current_date.year, current_date.month, current_date.day + ) + yield valid_date.date() + except ValueError: + pass + current_date -= timedelta(days=1) + + results = dao_fetch_stats_for_service_from_days(service_id, start_date, days) + + grouped_results = defaultdict(list) + for row in results: + notification_type, status, day, count = row + grouped_results[day.date()].append(row) + + for date in generate_date_range(start_date, days): + if date not in grouped_results: + grouped_results[date] = [] + + stats = {} + for day, rows in grouped_results.items(): + stats[day.strftime("%Y-%m-%d")] = statistics.format_statistics(rows) return stats @@ -262,23 +273,35 @@ def get_service_statistics_for_specific_days_by_user( ): start_date = datetime.strptime(start, "%Y-%m-%d").date() - if days == 1: - stats = {} - stats[start] = statistics.format_statistics( - dao_fetch_stats_for_service_from_day_for_user( - service_id, start_date, user_id - ) - ) - else: - stats = {} - for d in range(days): - new_date = start_date - timedelta(days=d) - key = new_date.strftime("%Y-%m-%d") - stats[key] = statistics.format_statistics( - dao_fetch_stats_for_service_from_day_for_user( - service_id, new_date, user_id + def generate_date_range(start_date, days): + current_date = start_date + end_date = start_date - timedelta(days=days) + while current_date > end_date: + try: + valid_date = datetime( + current_date.year, current_date.month, current_date.day ) - ) + yield valid_date.date() + except ValueError: + pass + current_date -= timedelta(days=1) + + results = dao_fetch_stats_for_service_from_days_for_user( + service_id, start_date, days, user_id + ) + + grouped_results = defaultdict(list) + for row in results: + notification_type, status, day, count = row + grouped_results[day.date()].append(row) + + for date in generate_date_range(start_date, days): + if date not in grouped_results: + grouped_results[date] = [] + + stats = {} + for day, rows in grouped_results.items(): + stats[day.strftime("%Y-%m-%d")] = statistics.format_statistics(rows) return stats @@ -732,19 +755,37 @@ def get_single_month_notification_stats_by_user(service_id, user_id): ) month_year = datetime(year, month, 10, 00, 00, 00) - days = get_number_of_days_for_month(year, month) start_date, end_date = get_month_start_and_end_date_in_utc(month_year) - stats = {} - for d in range(days): - new_date = start_date + timedelta(days=d) - if new_date <= end_date: - key = new_date.strftime("%Y-%m-%d") - stats[key] = statistics.format_statistics( - dao_fetch_stats_for_service_from_day_for_user( - service_id, new_date, user_id + def generate_date_range(start_date, end_date): + current_date = start_date + end_date = end_date + while current_date < end_date: + try: + valid_date = datetime( + current_date.year, current_date.month, current_date.day ) - ) + yield valid_date.date() + except ValueError: + pass + current_date += timedelta(days=1) + + results = dao_fetch_stats_for_service_from_days_for_user( + service_id, start_date, user_id + ) + + grouped_results = defaultdict(list) + for row in results: + notification_type, status, day, count = row + grouped_results[day.date()].append(row) + + for date in generate_date_range(start_date, end_date): + if date not in grouped_results: + grouped_results[date] = [] + + stats = {} + for day, rows in grouped_results.items(): + stats[day.strftime("%Y-%m-%d")] = statistics.format_statistics(rows) return jsonify(stats) @@ -763,17 +804,35 @@ def get_single_month_notification_stats_for_service(service_id): ) month_year = datetime(year, month, 10, 00, 00, 00) - days = get_number_of_days_for_month(year, month) start_date, end_date = get_month_start_and_end_date_in_utc(month_year) + def generate_date_range(start_date, end_date): + current_date = start_date + end_date = end_date + while current_date < end_date: + try: + valid_date = datetime( + current_date.year, current_date.month, current_date.day + ) + yield valid_date.date() + except ValueError: + pass + current_date += timedelta(days=1) + + results = dao_fetch_stats_for_service_from_days(service_id, start_date) + + grouped_results = defaultdict(list) + for row in results: + notification_type, status, day, count = row + grouped_results[day.date()].append(row) + + for date in generate_date_range(start_date, end_date): + if date not in grouped_results: + grouped_results[date] = [] + stats = {} - for d in range(days): - new_date = start_date + timedelta(days=d) - if new_date <= end_date: - key = new_date.strftime("%Y-%m-%d") - stats[key] = statistics.format_statistics( - dao_fetch_stats_for_service_from_day(service_id, new_date) - ) + for day, rows in grouped_results.items(): + stats[day.strftime("%Y-%m-%d")] = statistics.format_statistics(rows) return jsonify(stats) From 966f9b405059d95cd229fa05af90c5cd4ecde7e9 Mon Sep 17 00:00:00 2001 From: Anastasia Gradova Date: Thu, 20 Jun 2024 23:12:47 -0600 Subject: [PATCH 12/45] moved generate_date_range to date_util standardized the SQLAlchemy calls refactored the endpoints in service/rest.py --- app/dao/date_util.py | 22 +++++++++++ app/dao/services_dao.py | 14 ++++--- app/service/rest.py | 84 +++++++++++------------------------------ 3 files changed, 53 insertions(+), 67 deletions(-) diff --git a/app/dao/date_util.py b/app/dao/date_util.py index 7acc587aa..cac09dee2 100644 --- a/app/dao/date_util.py +++ b/app/dao/date_util.py @@ -71,3 +71,25 @@ def get_calendar_year_for_datetime(start_date): def get_number_of_days_for_month(year, month): return calendar.monthrange(year, month)[1] + + +def generate_date_range(start_date, end_date=None, days=0): + if end_date: + current_date = start_date + while current_date <= end_date: + try: + yield current_date.date() + except ValueError: + pass + current_date += timedelta(days=1) + elif days > 0: + end_date = start_date + timedelta(days=days) + current_date = start_date + while current_date < end_date: + try: + yield current_date.date() + except ValueError: + pass + current_date += timedelta(days=1) + else: + return "A start_date or number of days must be specified" diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 7b16aa8b3..6aa0d42f3 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -427,9 +427,9 @@ def dao_fetch_todays_stats_for_service(service_id): ) -def dao_fetch_stats_for_service_from_days(service_id, start, days): - start_date = get_midnight_in_utc(start) - end_date = get_midnight_in_utc(start + timedelta(days=days)) +def dao_fetch_stats_for_service_from_days(service_id, start_date, end_date): + start_date = get_midnight_in_utc(start_date) + end_date = get_midnight_in_utc(end_date) return ( db.session.query( @@ -453,9 +453,11 @@ def dao_fetch_stats_for_service_from_days(service_id, start, days): ) -def dao_fetch_stats_for_service_from_days_for_user(service_id, start, days, user_id): - start_date = get_midnight_in_utc(start) - end_date = get_midnight_in_utc(start + timedelta(days=days)) +def dao_fetch_stats_for_service_from_days_for_user( + service_id, start_date, end_date, user_id +): + start_date = get_midnight_in_utc(start_date) + end_date = get_midnight_in_utc(end_date) return ( db.session.query( diff --git a/app/service/rest.py b/app/service/rest.py index 204da0c80..ea59387d8 100644 --- a/app/service/rest.py +++ b/app/service/rest.py @@ -18,7 +18,11 @@ from app.dao.api_key_dao import ( save_model_api_key, ) from app.dao.dao_utils import dao_rollback, transaction -from app.dao.date_util import get_calendar_year, get_month_start_and_end_date_in_utc +from app.dao.date_util import ( + generate_date_range, + get_calendar_year, + get_month_start_and_end_date_in_utc, +) from app.dao.fact_notification_status_dao import ( fetch_monthly_template_usage_for_service, fetch_notification_status_for_service_by_month, @@ -222,29 +226,20 @@ def get_service_notification_statistics_by_day(service_id, start, days): def get_service_statistics_for_specific_days(service_id, start, days=1): - start_date = datetime.strptime(start, "%Y-%m-%d").date() + # start and end dates needs to be reversed because + # the end date is today and the start is x days in the past + # a day needs to be substracted to allow for today + end_date = datetime.strptime(start, "%Y-%m-%d") + start_date = end_date - timedelta(days=days - 1) - def generate_date_range(start_date, days): - current_date = start_date - end_date = start_date - timedelta(days=days) - while current_date > end_date: - try: - valid_date = datetime( - current_date.year, current_date.month, current_date.day - ) - yield valid_date.date() - except ValueError: - pass - current_date -= timedelta(days=1) - - results = dao_fetch_stats_for_service_from_days(service_id, start_date, days) + results = dao_fetch_stats_for_service_from_days(service_id, start_date, end_date) grouped_results = defaultdict(list) for row in results: notification_type, status, day, count = row grouped_results[day.date()].append(row) - for date in generate_date_range(start_date, days): + for date in generate_date_range(start_date, days=days): if date not in grouped_results: grouped_results[date] = [] @@ -271,23 +266,14 @@ def get_service_notification_statistics_by_day_by_user( def get_service_statistics_for_specific_days_by_user( service_id, user_id, start, days=1 ): - start_date = datetime.strptime(start, "%Y-%m-%d").date() - - def generate_date_range(start_date, days): - current_date = start_date - end_date = start_date - timedelta(days=days) - while current_date > end_date: - try: - valid_date = datetime( - current_date.year, current_date.month, current_date.day - ) - yield valid_date.date() - except ValueError: - pass - current_date -= timedelta(days=1) + # start and end dates needs to be reversed because + # the end date is today and the start is x days in the past + # a day needs to be substracted to allow for today + end_date = datetime.strptime(start, "%Y-%m-%d") + start_date = end_date - timedelta(days=days - 1) results = dao_fetch_stats_for_service_from_days_for_user( - service_id, start_date, days, user_id + service_id, start_date, end_date, user_id ) grouped_results = defaultdict(list) @@ -295,7 +281,9 @@ def get_service_statistics_for_specific_days_by_user( notification_type, status, day, count = row grouped_results[day.date()].append(row) - for date in generate_date_range(start_date, days): + print(grouped_results) + + for date in generate_date_range(start_date, days=days): if date not in grouped_results: grouped_results[date] = [] @@ -752,21 +740,8 @@ def get_single_month_notification_stats_by_user(service_id, user_id): month_year = datetime(year, month, 10, 00, 00, 00) start_date, end_date = get_month_start_and_end_date_in_utc(month_year) - def generate_date_range(start_date, end_date): - current_date = start_date - end_date = end_date - while current_date < end_date: - try: - valid_date = datetime( - current_date.year, current_date.month, current_date.day - ) - yield valid_date.date() - except ValueError: - pass - current_date += timedelta(days=1) - results = dao_fetch_stats_for_service_from_days_for_user( - service_id, start_date, user_id + service_id, start_date, end_date, user_id ) grouped_results = defaultdict(list) @@ -801,20 +776,7 @@ def get_single_month_notification_stats_for_service(service_id): month_year = datetime(year, month, 10, 00, 00, 00) start_date, end_date = get_month_start_and_end_date_in_utc(month_year) - def generate_date_range(start_date, end_date): - current_date = start_date - end_date = end_date - while current_date < end_date: - try: - valid_date = datetime( - current_date.year, current_date.month, current_date.day - ) - yield valid_date.date() - except ValueError: - pass - current_date += timedelta(days=1) - - results = dao_fetch_stats_for_service_from_days(service_id, start_date) + results = dao_fetch_stats_for_service_from_days(service_id, start_date, end_date) grouped_results = defaultdict(list) for row in results: From d3d2610578b94e05407bbe1d3c1673c26bdb8930 Mon Sep 17 00:00:00 2001 From: Anastasia Gradova Date: Fri, 21 Jun 2024 14:38:54 -0600 Subject: [PATCH 13/45] correct timedelta for midnight that offest calculation for today --- app/dao/services_dao.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 6aa0d42f3..dcf2536eb 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -429,7 +429,7 @@ def dao_fetch_todays_stats_for_service(service_id): def dao_fetch_stats_for_service_from_days(service_id, start_date, end_date): start_date = get_midnight_in_utc(start_date) - end_date = get_midnight_in_utc(end_date) + end_date = get_midnight_in_utc(end_date + timedelta(days=1)) return ( db.session.query( @@ -457,7 +457,7 @@ def dao_fetch_stats_for_service_from_days_for_user( service_id, start_date, end_date, user_id ): start_date = get_midnight_in_utc(start_date) - end_date = get_midnight_in_utc(end_date) + end_date = get_midnight_in_utc(end_date + timedelta(days=1)) return ( db.session.query( From 47c89647665b4cb464136c11f55b0e11814512f6 Mon Sep 17 00:00:00 2001 From: Anastasia Gradova Date: Fri, 21 Jun 2024 16:58:09 -0600 Subject: [PATCH 14/45] fetch_notification_status_for_service_by_month altered to use NotificationAllTimeView like the other endpoints --- app/dao/fact_notification_status_dao.py | 22 +++++++++++----------- app/dao/services_dao.py | 4 ++-- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/app/dao/fact_notification_status_dao.py b/app/dao/fact_notification_status_dao.py index 22c87fe83..f2769d2cd 100644 --- a/app/dao/fact_notification_status_dao.py +++ b/app/dao/fact_notification_status_dao.py @@ -84,21 +84,21 @@ def update_fact_notification_status(process_day, notification_type, service_id): def fetch_notification_status_for_service_by_month(start_date, end_date, service_id): return ( db.session.query( - func.date_trunc("month", FactNotificationStatus.local_date).label("month"), - FactNotificationStatus.notification_type, - FactNotificationStatus.notification_status, - func.sum(FactNotificationStatus.notification_count).label("count"), + func.date_trunc("month", NotificationAllTimeView.created_at).label("month"), + NotificationAllTimeView.notification_type, + NotificationAllTimeView.status.label('notification_status'), + func.count(NotificationAllTimeView.id).label("count"), ) .filter( - FactNotificationStatus.service_id == service_id, - FactNotificationStatus.local_date >= start_date, - FactNotificationStatus.local_date < end_date, - FactNotificationStatus.key_type != KeyType.TEST, + NotificationAllTimeView.service_id == service_id, + NotificationAllTimeView.created_at >= start_date, + NotificationAllTimeView.created_at < end_date, + NotificationAllTimeView.key_type != KeyType.TEST, ) .group_by( - func.date_trunc("month", FactNotificationStatus.local_date).label("month"), - FactNotificationStatus.notification_type, - FactNotificationStatus.notification_status, + func.date_trunc("month", NotificationAllTimeView.created_at).label("month"), + NotificationAllTimeView.notification_type, + NotificationAllTimeView.status, ) .all() ) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index dcf2536eb..74f8094a8 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -442,7 +442,7 @@ def dao_fetch_stats_for_service_from_days(service_id, start_date, end_date): NotificationAllTimeView.service_id == service_id, NotificationAllTimeView.key_type != KeyType.TEST, NotificationAllTimeView.created_at >= start_date, - NotificationAllTimeView.created_at <= end_date, + NotificationAllTimeView.created_at < end_date, ) .group_by( NotificationAllTimeView.notification_type, @@ -470,7 +470,7 @@ def dao_fetch_stats_for_service_from_days_for_user( NotificationAllTimeView.service_id == service_id, NotificationAllTimeView.key_type != KeyType.TEST, NotificationAllTimeView.created_at >= start_date, - NotificationAllTimeView.created_at <= end_date, + NotificationAllTimeView.created_at < end_date, NotificationAllTimeView.created_by_id == user_id, ) .group_by( From b2e5522d09ec61a38262efac6770099363930cee Mon Sep 17 00:00:00 2001 From: Anastasia Gradova Date: Mon, 24 Jun 2024 17:58:37 -0600 Subject: [PATCH 15/45] Corrected test cases for new stats endpoints --- app/dao/fact_notification_status_dao.py | 2 +- .../dao/test_fact_notification_status_dao.py | 59 +++-- tests/app/service/test_statistics_rest.py | 220 ++++++++++-------- 3 files changed, 161 insertions(+), 120 deletions(-) diff --git a/app/dao/fact_notification_status_dao.py b/app/dao/fact_notification_status_dao.py index f2769d2cd..a810ff0db 100644 --- a/app/dao/fact_notification_status_dao.py +++ b/app/dao/fact_notification_status_dao.py @@ -86,7 +86,7 @@ def fetch_notification_status_for_service_by_month(start_date, end_date, service db.session.query( func.date_trunc("month", NotificationAllTimeView.created_at).label("month"), NotificationAllTimeView.notification_type, - NotificationAllTimeView.status.label('notification_status'), + NotificationAllTimeView.status.label("notification_status"), func.count(NotificationAllTimeView.id).label("count"), ) .filter( diff --git a/tests/app/dao/test_fact_notification_status_dao.py b/tests/app/dao/test_fact_notification_status_dao.py index 4c7030b2e..dc46de45d 100644 --- a/tests/app/dao/test_fact_notification_status_dao.py +++ b/tests/app/dao/test_fact_notification_status_dao.py @@ -33,31 +33,44 @@ def test_fetch_notification_status_for_service_by_month(notify_db_session): service_1 = create_service(service_name="service_1") service_2 = create_service(service_name="service_2") - create_ft_notification_status( - date(2018, 1, 1), NotificationType.SMS, service_1, count=4 - ) - create_ft_notification_status( - date(2018, 1, 2), NotificationType.SMS, service_1, count=10 - ) - create_ft_notification_status( - date(2018, 1, 2), - NotificationType.SMS, - service_1, - notification_status=NotificationStatus.CREATED, - ) - create_ft_notification_status(date(2018, 1, 3), NotificationType.EMAIL, service_1) + create_template(service=service_1) + create_template(service=service_1, template_type=TemplateType.EMAIL) + # not the service being tested + create_template(service=service_2) - create_ft_notification_status(date(2018, 2, 2), NotificationType.SMS, service_1) + # loop messages for the month + for x in range(0, 14): + create_notification( + service_1.templates[0], + created_at=datetime(2018, 1, 1, 1, x, 0), + status=NotificationStatus.DELIVERED, + ) + create_notification( + service_1.templates[0], created_at=datetime(2018, 1, 1, 1, 1, 0) + ) + create_notification( + service_1.templates[1], + created_at=datetime(2018, 1, 1, 1, 1, 0), + status=NotificationStatus.DELIVERED, + ) + create_notification( + service_1.templates[0], + created_at=datetime(2018, 2, 1, 1, 1, 0), + status=NotificationStatus.DELIVERED, + ) - # not included - too early - create_ft_notification_status(date(2017, 12, 31), NotificationType.SMS, service_1) - # not included - too late - create_ft_notification_status(date(2017, 3, 1), NotificationType.SMS, service_1) - # not included - wrong service - create_ft_notification_status(date(2018, 1, 3), NotificationType.SMS, service_2) - # not included - test keys - create_ft_notification_status( - date(2018, 1, 3), NotificationType.SMS, service_1, key_type=KeyType.TEST + # not the right month + create_notification( + service_1.templates[0], + created_at=datetime(2018, 4, 1, 1, 1, 0), + status=NotificationStatus.DELIVERED, + ) + + # not the right service + create_notification( + service_2.templates[0], + created_at=datetime(2018, 2, 1, 1, 1, 0), + status=NotificationStatus.DELIVERED, ) results = sorted( diff --git a/tests/app/service/test_statistics_rest.py b/tests/app/service/test_statistics_rest.py index 2163f8f36..9769a678e 100644 --- a/tests/app/service/test_statistics_rest.py +++ b/tests/app/service/test_statistics_rest.py @@ -234,17 +234,36 @@ def test_get_monthly_notification_stats_returns_stats(admin_request, sample_serv sms_t2 = create_template(sample_service) email_template = create_template(sample_service, template_type=TemplateType.EMAIL) - create_ft_notification_status(datetime(2016, 6, 1), template=sms_t1) - create_ft_notification_status(datetime(2016, 6, 2), template=sms_t1) - - create_ft_notification_status(datetime(2016, 7, 1), template=sms_t1) - create_ft_notification_status(datetime(2016, 7, 1), template=sms_t2) - create_ft_notification_status( - datetime(2016, 7, 1), - template=sms_t1, - notification_status=NotificationStatus.CREATED, + create_notification( + sms_t1, + created_at=datetime(2016, 6, 1, 1, 1, 0), + status=NotificationStatus.DELIVERED, + ) + create_notification( + sms_t1, + created_at=datetime(2016, 6, 2, 1, 1, 0), + status=NotificationStatus.DELIVERED, + ) + create_notification( + sms_t1, + created_at=datetime(2016, 7, 1, 1, 1, 0), + status=NotificationStatus.DELIVERED, + ) + create_notification( + sms_t2, + created_at=datetime(2016, 7, 1, 1, 1, 0), + status=NotificationStatus.DELIVERED, + ) + create_notification( + sms_t1, + created_at=datetime(2016, 7, 1, 1, 1, 0), + status=NotificationStatus.CREATED, + ) + create_notification( + email_template, + created_at=datetime(2016, 7, 1, 1, 1, 0), + status=NotificationStatus.DELIVERED, ) - create_ft_notification_status(datetime(2016, 7, 1), template=email_template) response = admin_request.get( "service.get_monthly_notification_stats", @@ -275,85 +294,94 @@ def test_get_monthly_notification_stats_returns_stats(admin_request, sample_serv } -@freeze_time("2016-06-05 12:00:00") -def test_get_monthly_notification_stats_combines_todays_data_and_historic_stats( - admin_request, sample_template -): - create_ft_notification_status( - datetime(2016, 5, 1, 12), - template=sample_template, - count=1, - ) - create_ft_notification_status( - datetime(2016, 6, 1, 12), - template=sample_template, - notification_status=NotificationStatus.CREATED, - count=2, - ) # noqa +# Test removed because new endpoint uses the view which combines this data +# @freeze_time("2016-06-05 12:00:00") +# def test_get_monthly_notification_stats_combines_todays_data_and_historic_stats( +# admin_request, sample_template +# ): +# create_ft_notification_status( +# datetime(2016, 5, 1, 12), +# template=sample_template, +# count=1, +# ) +# create_ft_notification_status( +# datetime(2016, 6, 1, 12), +# template=sample_template, +# notification_status=NotificationStatus.CREATED, +# count=2, +# ) # noqa - create_notification( - sample_template, - created_at=datetime(2016, 6, 5, 12), - status=NotificationStatus.CREATED, - ) - create_notification( - sample_template, - created_at=datetime(2016, 6, 5, 12), - status=NotificationStatus.DELIVERED, - ) +# create_notification( +# sample_template, +# created_at=datetime(2016, 6, 5, 12), +# status=NotificationStatus.CREATED, +# ) +# create_notification( +# sample_template, +# created_at=datetime(2016, 6, 5, 12), +# status=NotificationStatus.DELIVERED, +# ) - # this doesn't get returned in the stats because it is old - it should be in ft_notification_status by now - create_notification( - sample_template, - created_at=datetime(2016, 6, 4, 12), - status=NotificationStatus.SENDING, - ) +# # this doesn't get returned in the stats because it is old - it should be in ft_notification_status by now +# create_notification( +# sample_template, +# created_at=datetime(2016, 6, 4, 12), +# status=NotificationStatus.SENDING, +# ) - response = admin_request.get( - "service.get_monthly_notification_stats", - service_id=sample_template.service_id, - year=2016, - ) +# response = admin_request.get( +# "service.get_monthly_notification_stats", +# service_id=sample_template.service_id, +# year=2016, +# ) - assert len(response["data"]) == 6 # January to June - assert response["data"]["2016-05"] == { - NotificationType.SMS: { - NotificationStatus.DELIVERED: 1, - StatisticsType.REQUESTED: 1, - }, - NotificationType.EMAIL: {}, - } - assert response["data"]["2016-06"] == { - NotificationType.SMS: { - # combines the stats from the historic ft_notification_status and the current notifications - NotificationStatus.CREATED: 3, - NotificationStatus.DELIVERED: 1, - StatisticsType.REQUESTED: 4, - }, - NotificationType.EMAIL: {}, - } +# assert len(response["data"]) == 6 # January to June +# assert response["data"]["2016-05"] == { +# NotificationType.SMS: { +# NotificationStatus.DELIVERED: 1, +# StatisticsType.REQUESTED: 1, +# }, +# NotificationType.EMAIL: {}, +# } +# assert response["data"]["2016-06"] == { +# NotificationType.SMS: { +# # combines the stats from the historic ft_notification_status and the current notifications +# NotificationStatus.CREATED: 3, +# NotificationStatus.DELIVERED: 1, +# StatisticsType.REQUESTED: 4, +# }, +# NotificationType.EMAIL: {}, +# } def test_get_monthly_notification_stats_ignores_test_keys( admin_request, sample_service ): - create_ft_notification_status( - datetime(2016, 6, 1), - service=sample_service, + create_template(service=sample_service) + + create_notification( + sample_service.templates[0], + created_at=datetime(2016, 6, 1, 1, 1, 0), key_type=KeyType.NORMAL, - count=1, + status=NotificationStatus.DELIVERED, ) - create_ft_notification_status( - datetime(2016, 6, 1), - service=sample_service, + create_notification( + sample_service.templates[0], + created_at=datetime(2016, 6, 2, 1, 1, 0), + key_type=KeyType.NORMAL, + status=NotificationStatus.DELIVERED, + ) + create_notification( + sample_service.templates[0], + created_at=datetime(2016, 6, 1, 1, 1, 0), key_type=KeyType.TEAM, - count=2, + status=NotificationStatus.DELIVERED, ) - create_ft_notification_status( - datetime(2016, 6, 1), - service=sample_service, + create_notification( + sample_service.templates[0], + created_at=datetime(2016, 6, 1, 1, 1, 0), key_type=KeyType.TEST, - count=4, + status=NotificationStatus.DELIVERED, ) response = admin_request.get( @@ -370,21 +398,21 @@ def test_get_monthly_notification_stats_ignores_test_keys( def test_get_monthly_notification_stats_checks_dates(admin_request, sample_service): t = create_template(sample_service) - # create_ft_notification_status(datetime(2016, 3, 31), template=t, notification_status='created') - create_ft_notification_status( - datetime(2016, 4, 2), - template=t, - notification_status=NotificationStatus.SENDING, + + create_notification( + t, + created_at=datetime(2016, 4, 2), + status=NotificationStatus.SENDING, ) - create_ft_notification_status( - datetime(2017, 3, 31), - template=t, - notification_status=NotificationStatus.DELIVERED, + create_notification( + t, + created_at=datetime(2017, 3, 31), + status=NotificationStatus.DELIVERED, ) - create_ft_notification_status( - datetime(2017, 4, 11), - template=t, - notification_status=NotificationStatus.PERMANENT_FAILURE, + create_notification( + t, + created_at=datetime(2017, 4, 11), + status=NotificationStatus.PERMANENT_FAILURE, ) response = admin_request.get( @@ -411,15 +439,15 @@ def test_get_monthly_notification_stats_only_gets_for_one_service( templates = [create_template(services[0]), create_template(services[1])] - create_ft_notification_status( - datetime(2016, 6, 1), - template=templates[0], - notification_status=NotificationStatus.CREATED, + create_notification( + templates[0], + created_at=datetime(2016, 6, 1), + status=NotificationStatus.CREATED, ) - create_ft_notification_status( - datetime(2016, 6, 1), - template=templates[1], - notification_status=NotificationStatus.DELIVERED, + create_notification( + templates[1], + created_at=datetime(2016, 6, 1), + status=NotificationStatus.DELIVERED, ) response = admin_request.get( From 8e6a6d42dbfd8025dcdc8d5cb6368710d2dbdde1 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Wed, 26 Jun 2024 11:59:17 -0700 Subject: [PATCH 16/45] remove task --- app/aws/s3.py | 40 ---------------------------------------- app/celery/tasks.py | 29 ----------------------------- app/user/rest.py | 7 +++++++ 3 files changed, 7 insertions(+), 69 deletions(-) diff --git a/app/aws/s3.py b/app/aws/s3.py index 8435b4938..9466e6cce 100644 --- a/app/aws/s3.py +++ b/app/aws/s3.py @@ -1,5 +1,4 @@ import re -import uuid import botocore from boto3 import Session @@ -8,7 +7,6 @@ from flask import current_app from app import redis_store from app.clients import AWS_CLIENT_CONFIG -from notifications_utils.s3 import s3upload as utils_s3upload FILE_LOCATION_STRUCTURE = "service-{}-notify/{}.csv" @@ -21,31 +19,11 @@ JOBS_CACHE_HITS = "JOBS_CACHE_HITS" JOBS_CACHE_MISSES = "JOBS_CACHE_MISSES" -def get_csv_location(service_id, upload_id): - return ( - current_app.config["CSV_UPLOAD_BUCKET"]["bucket"], - FILE_LOCATION_STRUCTURE.format(service_id, upload_id), - current_app.config["CSV_UPLOAD_BUCKET"]["access_key_id"], - current_app.config["CSV_UPLOAD_BUCKET"]["secret_access_key"], - current_app.config["CSV_UPLOAD_BUCKET"]["region"], - ) - - def get_s3_file(bucket_name, file_location, access_key, secret_key, region): s3_file = get_s3_object(bucket_name, file_location, access_key, secret_key, region) return s3_file.get()["Body"].read().decode("utf-8") -def get_file_from_s3(file_location): - return get_s3_file( - current_app.config["CSV_UPLOAD_BUCKET"]["bucket"], - file_location, - current_app.config["CSV_UPLOAD_BUCKET"]["access_key_id"], - current_app.config["CSV_UPLOAD_BUCKET"]["secret_access_key"], - current_app.config["CSV_UPLOAD_BUCKET"]["region"], - ) - - def get_s3_object(bucket_name, file_location, access_key, secret_key, region): session = Session( aws_access_key_id=access_key, @@ -275,21 +253,3 @@ def remove_csv_object(object_key): current_app.config["CSV_UPLOAD_BUCKET"]["region"], ) return obj.delete() - - -def s3upload(service_id, filedata, upload_id=None): - - if upload_id is None: - upload_id = str(uuid.uuid4()) - bucket_name, file_location, access_key, secret_key, region = get_csv_location( - service_id, upload_id - ) - utils_s3upload( - filedata=filedata["data"], - region=region, - bucket_name=bucket_name, - file_location=file_location, - access_key=access_key, - secret_key=secret_key, - ) - return upload_id diff --git a/app/celery/tasks.py b/app/celery/tasks.py index ecaa8479b..f0d036549 100644 --- a/app/celery/tasks.py +++ b/app/celery/tasks.py @@ -1,5 +1,4 @@ import json -import os from flask import current_app from requests import HTTPError, RequestException, request @@ -19,7 +18,6 @@ from app.dao.service_email_reply_to_dao import dao_get_reply_to_by_id from app.dao.service_inbound_api_dao import get_service_inbound_api_for_service from app.dao.service_sms_sender_dao import dao_get_service_sms_senders_by_id from app.dao.templates_dao import dao_get_template_by_id -from app.dao.users_dao import dao_report_users from app.enums import JobStatus, KeyType, NotificationType from app.errors import TotalRequestsError from app.notifications.process_notifications import persist_notification @@ -481,30 +479,3 @@ def process_incomplete_job(job_id): process_row(row, template, job, job.service, sender_id=sender_id) job_complete(job, resumed=True) - - -@notify_celery.task(name="report-all-users") -def report_all_users(): - """ - This is to support the platform admin's ability to view all user data. - It runs once per night and is stored in - bucket/service-all-users-report-{env}-notify/all-users-report-{env}.csv - - When the front end is ready, it can just download from there. - """ - users = dao_report_users() - csv_text = "NAME,EMAIL_ADDRESS,MOBILE_NUMBER,SERVICE\n" - for user in users: - row = f"{user[0]},{user[1]},{user[2]},{user[3]}\n" - csv_text = f"{csv_text}{row}" - my_env = os.getenv("NOTIFY_ENVIRONMENT") - report_name = f"all-users-report-{my_env}" - file_data = {} - file_data["data"] = csv_text - object_key = s3.FILE_LOCATION_STRUCTURE.format(report_name, report_name) - s3.remove_csv_object(object_key) - s3.s3upload(report_name, file_data, report_name) - - # prove that it works - x = s3.get_file_from_s3(object_key) - print(f"!!!!!!!DOWNLOADED {x}") diff --git a/app/user/rest.py b/app/user/rest.py index 049549f2c..dd714ce57 100644 --- a/app/user/rest.py +++ b/app/user/rest.py @@ -18,6 +18,7 @@ from app.dao.users_dao import ( create_secret_code, create_user_code, dao_archive_user, + dao_report_users, get_login_gov_user, get_user_and_accounts, get_user_by_email, @@ -667,6 +668,12 @@ def update_password(user_id): return jsonify(data=user.serialize()), 200 +@user_blueprint.route("/report-all-users", methods=["GET"]) +def report_all_users(): + users = dao_report_users() + return jsonify(data=users.serialize()), 200 + + @user_blueprint.route("//organizations-and-services", methods=["GET"]) def get_organizations_and_services_for_user(user_id): user = get_user_and_accounts(user_id) From f6f67b6d46f408783a8af3cc6185ef0d5f027d62 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Jun 2024 21:11:42 +0000 Subject: [PATCH 17/45] Bump botocore from 1.34.133 to 1.34.134 Bumps [botocore](https://github.com/boto/botocore) from 1.34.133 to 1.34.134. - [Changelog](https://github.com/boto/botocore/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/botocore/compare/1.34.133...1.34.134) --- updated-dependencies: - dependency-name: botocore dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- poetry.lock | 16 ++++++++-------- pyproject.toml | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/poetry.lock b/poetry.lock index 1c5be2257..c2fa76e11 100644 --- a/poetry.lock +++ b/poetry.lock @@ -204,17 +204,17 @@ tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "p [[package]] name = "awscli" -version = "1.33.15" +version = "1.33.16" description = "Universal Command Line Environment for AWS." optional = false python-versions = ">=3.8" files = [ - {file = "awscli-1.33.15-py3-none-any.whl", hash = "sha256:5a8d7e68a4cf68afc9d9ba4bef511526eb71027360f95a1080d39158bc930083"}, - {file = "awscli-1.33.15.tar.gz", hash = "sha256:54a8089edb6756da46addcfcd56fdca21307a121216a81ef542e17b284cbe9c9"}, + {file = "awscli-1.33.16-py3-none-any.whl", hash = "sha256:8eef82e6c5c3d1f6c881ed4558eaf790c6e433670d4fb5cac622ed5a74d54c98"}, + {file = "awscli-1.33.16.tar.gz", hash = "sha256:5550ca894ab66974061ad3f8fef7a9bc579e5dd8f25a97eebf356e9d8abd5907"}, ] [package.dependencies] -botocore = "1.34.133" +botocore = "1.34.134" colorama = ">=0.2.5,<0.4.7" docutils = ">=0.10,<0.17" PyYAML = ">=3.10,<6.1" @@ -422,13 +422,13 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.34.133" +version = "1.34.134" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">=3.8" files = [ - {file = "botocore-1.34.133-py3-none-any.whl", hash = "sha256:f269dad8e17432d2527b97ed9f1fd30ec8dc705f8b818957170d1af484680ef2"}, - {file = "botocore-1.34.133.tar.gz", hash = "sha256:5ea609aa4831a6589e32eef052a359ad8d7311733b4d86a9d35dab4bd3ec80ff"}, + {file = "botocore-1.34.134-py3-none-any.whl", hash = "sha256:45219e00639755f92569b29f8f279d5dde721494791412c1f7026a3779e8d9f4"}, + {file = "botocore-1.34.134.tar.gz", hash = "sha256:e29c299599426ed16dd2d4c1e20eef784f96b15e1850ebbc59a3250959285b95"}, ] [package.dependencies] @@ -4752,4 +4752,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.12.2" -content-hash = "b1b4bfbfdc1f5cc9ae9d090f35b235a62e9dbabc683a5b5a1d0d414605219b48" +content-hash = "2c1d04b521f540a0d3580fa483359628de6ef2ef4c61fc0f7f32f8bb79e65f42" diff --git a/pyproject.toml b/pyproject.toml index 2ea004520..72da28cb9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ alembic = "==1.13.1" amqp = "==5.2.0" beautifulsoup4 = "==4.12.3" boto3 = "^1.34.131" -botocore = "^1.34.133" +botocore = "^1.34.134" cachetools = "==5.3.3" celery = {version = "==5.4.0", extras = ["redis"]} certifi = ">=2022.12.7" From 85aaa2952d3f922172d63b5e8a670fc5f3764564 Mon Sep 17 00:00:00 2001 From: Cliff Hill Date: Thu, 27 Jun 2024 10:56:32 -0400 Subject: [PATCH 18/45] Removed old redis from terraform. Signed-off-by: Cliff Hill --- terraform/demo/main.tf | 9 --------- terraform/production/main.tf | 9 --------- terraform/staging/main.tf | 9 --------- 3 files changed, 27 deletions(-) diff --git a/terraform/demo/main.tf b/terraform/demo/main.tf index def3903a0..ea0f259e4 100644 --- a/terraform/demo/main.tf +++ b/terraform/demo/main.tf @@ -21,15 +21,6 @@ module "database" { rds_plan_name = "micro-psql" } -module "redis" { # default v6.2; delete after v7.0 resource is bound - source = "github.com/GSA-TTS/terraform-cloudgov//redis?ref=v1.0.0" - - cf_org_name = local.cf_org_name - cf_space_name = local.cf_space_name - name = "${local.app_name}-redis-${local.env}" - redis_plan_name = "redis-dev" -} - module "redis-v70" { source = "github.com/GSA-TTS/terraform-cloudgov//redis?ref=v1.0.0" diff --git a/terraform/production/main.tf b/terraform/production/main.tf index 28b2bf2d6..45cf7a5b8 100644 --- a/terraform/production/main.tf +++ b/terraform/production/main.tf @@ -21,15 +21,6 @@ module "database" { rds_plan_name = "small-psql-redundant" } -module "redis" { # default v6.2; delete after v7.0 resource is bound - source = "github.com/GSA-TTS/terraform-cloudgov//redis?ref=v1.0.0" - - cf_org_name = local.cf_org_name - cf_space_name = local.cf_space_name - name = "${local.app_name}-redis-${local.env}" - redis_plan_name = "redis-3node-large" -} - module "redis-v70" { source = "github.com/GSA-TTS/terraform-cloudgov//redis?ref=v1.0.0" diff --git a/terraform/staging/main.tf b/terraform/staging/main.tf index 3cc0358a5..4fdbf9e38 100644 --- a/terraform/staging/main.tf +++ b/terraform/staging/main.tf @@ -21,15 +21,6 @@ module "database" { rds_plan_name = "micro-psql" } -module "redis" { # default v6.2; delete after v7.0 resource is bound - source = "github.com/GSA-TTS/terraform-cloudgov//redis?ref=v1.0.0" - - cf_org_name = local.cf_org_name - cf_space_name = local.cf_space_name - name = "${local.app_name}-redis-${local.env}" - redis_plan_name = "redis-dev" -} - module "redis-v70" { source = "github.com/GSA-TTS/terraform-cloudgov//redis?ref=v1.0.0" From 76f31824e88b41d047d928af4e3b2c1fe8051517 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 27 Jun 2024 19:06:07 +0000 Subject: [PATCH 19/45] Bump s3transfer from 0.10.1 to 0.10.2 Bumps [s3transfer](https://github.com/boto/s3transfer) from 0.10.1 to 0.10.2. - [Changelog](https://github.com/boto/s3transfer/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/s3transfer/compare/0.10.1...0.10.2) --- updated-dependencies: - dependency-name: s3transfer dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- poetry.lock | 10 +++++----- pyproject.toml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/poetry.lock b/poetry.lock index c2fa76e11..05e104dbd 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3989,13 +3989,13 @@ pyasn1 = ">=0.1.3" [[package]] name = "s3transfer" -version = "0.10.1" +version = "0.10.2" description = "An Amazon S3 Transfer Manager" optional = false -python-versions = ">= 3.8" +python-versions = ">=3.8" files = [ - {file = "s3transfer-0.10.1-py3-none-any.whl", hash = "sha256:ceb252b11bcf87080fb7850a224fb6e05c8a776bab8f2b64b7f25b969464839d"}, - {file = "s3transfer-0.10.1.tar.gz", hash = "sha256:5683916b4c724f799e600f41dd9e10a9ff19871bf87623cc8f491cb4f5fa0a19"}, + {file = "s3transfer-0.10.2-py3-none-any.whl", hash = "sha256:eca1c20de70a39daee580aef4986996620f365c4e0fda6a86100231d62f1bf69"}, + {file = "s3transfer-0.10.2.tar.gz", hash = "sha256:0711534e9356d3cc692fdde846b4a1e4b0cb6519971860796e6bc4c7aea00ef6"}, ] [package.dependencies] @@ -4752,4 +4752,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.12.2" -content-hash = "2c1d04b521f540a0d3580fa483359628de6ef2ef4c61fc0f7f32f8bb79e65f42" +content-hash = "e731109e7d83aef0f517b6e84d254076a5ad52d612b9094b364ecb8333efb6b5" diff --git a/pyproject.toml b/pyproject.toml index 72da28cb9..bc98ecd70 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,7 +70,7 @@ markupsafe = "^2.1.5" pycparser = "^2.22" python-dateutil = "^2.9.0.post0" pyyaml = "^6.0.1" -s3transfer = "^0.10.1" +s3transfer = "^0.10.2" six = "^1.16.0" urllib3 = "^2.2.2" webencodings = "^0.5.1" From c0134d94311f9a2b97d3884450b032c7c90a256e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 27 Jun 2024 21:35:07 +0000 Subject: [PATCH 20/45] Bump alembic from 1.13.1 to 1.13.2 Bumps [alembic](https://github.com/sqlalchemy/alembic) from 1.13.1 to 1.13.2. - [Release notes](https://github.com/sqlalchemy/alembic/releases) - [Changelog](https://github.com/sqlalchemy/alembic/blob/main/CHANGES) - [Commits](https://github.com/sqlalchemy/alembic/commits) --- updated-dependencies: - dependency-name: alembic dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- poetry.lock | 8 ++++---- pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index 05e104dbd..0d5ece290 100644 --- a/poetry.lock +++ b/poetry.lock @@ -111,13 +111,13 @@ frozenlist = ">=1.1.0" [[package]] name = "alembic" -version = "1.13.1" +version = "1.13.2" description = "A database migration tool for SQLAlchemy." optional = false python-versions = ">=3.8" files = [ - {file = "alembic-1.13.1-py3-none-any.whl", hash = "sha256:2edcc97bed0bd3272611ce3a98d98279e9c209e7186e43e75bbb1b2bdfdbcc43"}, - {file = "alembic-1.13.1.tar.gz", hash = "sha256:4932c8558bf68f2ee92b9bbcb8218671c627064d5b08939437af6d77dc05e595"}, + {file = "alembic-1.13.2-py3-none-any.whl", hash = "sha256:6b8733129a6224a9a711e17c99b08462dbf7cc9670ba8f2e2ae9af860ceb1953"}, + {file = "alembic-1.13.2.tar.gz", hash = "sha256:1ff0ae32975f4fd96028c39ed9bb3c867fe3af956bd7bb37343b54c9fe7445ef"}, ] [package.dependencies] @@ -4752,4 +4752,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.12.2" -content-hash = "e731109e7d83aef0f517b6e84d254076a5ad52d612b9094b364ecb8333efb6b5" +content-hash = "6fdbf4f75ef649c3f050d0e67fe3f9437191de015a1191cd8949abb9c47f2223" diff --git a/pyproject.toml b/pyproject.toml index bc98ecd70..eb8b4c39e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ readme = "README.md" [tool.poetry.dependencies] python = "^3.12.2" -alembic = "==1.13.1" +alembic = "==1.13.2" amqp = "==5.2.0" beautifulsoup4 = "==4.12.3" boto3 = "^1.34.131" From f9b42da683f188f53d875ffc8fcee57aa0bd81fe Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 27 Jun 2024 21:54:35 +0000 Subject: [PATCH 21/45] Bump faker from 25.8.0 to 26.0.0 Bumps [faker](https://github.com/joke2k/faker) from 25.8.0 to 26.0.0. - [Release notes](https://github.com/joke2k/faker/releases) - [Changelog](https://github.com/joke2k/faker/blob/master/CHANGELOG.md) - [Commits](https://github.com/joke2k/faker/compare/v25.8.0...v26.0.0) --- updated-dependencies: - dependency-name: faker dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- poetry.lock | 8 ++++---- pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index 0d5ece290..c046dccc5 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1261,13 +1261,13 @@ tests = ["coverage", "coveralls", "dill", "mock", "nose"] [[package]] name = "faker" -version = "25.8.0" +version = "26.0.0" description = "Faker is a Python package that generates fake data for you." optional = false python-versions = ">=3.8" files = [ - {file = "Faker-25.8.0-py3-none-any.whl", hash = "sha256:4c40b34a9c569018d4f9d6366d71a4da8a883d5ddf2b23197be5370f29b7e1b6"}, - {file = "Faker-25.8.0.tar.gz", hash = "sha256:bdec5f2fb057d244ebef6e0ed318fea4dcbdf32c3a1a010766fc45f5d68fc68d"}, + {file = "Faker-26.0.0-py3-none-any.whl", hash = "sha256:886ee28219be96949cd21ecc96c4c742ee1680e77f687b095202c8def1a08f06"}, + {file = "Faker-26.0.0.tar.gz", hash = "sha256:0f60978314973de02c00474c2ae899785a42b2cf4f41b7987e93c132a2b8a4a9"}, ] [package.dependencies] @@ -4752,4 +4752,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.12.2" -content-hash = "6fdbf4f75ef649c3f050d0e67fe3f9437191de015a1191cd8949abb9c47f2223" +content-hash = "bf8c577b2f05c2cfbdea05e5006b1685b94981ca8b3156ac9c39d3c8a399a72e" diff --git a/pyproject.toml b/pyproject.toml index eb8b4c39e..de04706dc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,7 +48,7 @@ pyjwt = "==2.8.0" python-dotenv = "==1.0.1" sqlalchemy = "==2.0.31" werkzeug = "^3.0.3" -faker = "^25.8.0" +faker = "^26.0.0" async-timeout = "^4.0.3" bleach = "^6.1.0" geojson = "^3.1.0" From 821c303d9de1466f384489aaafa9d6f96a56bafb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 28 Jun 2024 14:21:03 +0000 Subject: [PATCH 22/45] Bump flake8 from 7.0.0 to 7.1.0 Bumps [flake8](https://github.com/pycqa/flake8) from 7.0.0 to 7.1.0. - [Commits](https://github.com/pycqa/flake8/compare/7.0.0...7.1.0) --- updated-dependencies: - dependency-name: flake8 dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- poetry.lock | 16 ++++++++-------- pyproject.toml | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/poetry.lock b/poetry.lock index c046dccc5..b25e061bd 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1305,18 +1305,18 @@ typing = ["typing-extensions (>=4.8)"] [[package]] name = "flake8" -version = "7.0.0" +version = "7.1.0" description = "the modular source code checker: pep8 pyflakes and co" optional = false python-versions = ">=3.8.1" files = [ - {file = "flake8-7.0.0-py2.py3-none-any.whl", hash = "sha256:a6dfbb75e03252917f2473ea9653f7cd799c3064e54d4c8140044c5c065f53c3"}, - {file = "flake8-7.0.0.tar.gz", hash = "sha256:33f96621059e65eec474169085dc92bf26e7b2d47366b70be2f67ab80dc25132"}, + {file = "flake8-7.1.0-py2.py3-none-any.whl", hash = "sha256:2e416edcc62471a64cea09353f4e7bdba32aeb079b6e360554c659a122b1bc6a"}, + {file = "flake8-7.1.0.tar.gz", hash = "sha256:48a07b626b55236e0fb4784ee69a465fbf59d79eec1f5b4785c3d3bc57d17aa5"}, ] [package.dependencies] mccabe = ">=0.7.0,<0.8.0" -pycodestyle = ">=2.11.0,<2.12.0" +pycodestyle = ">=2.12.0,<2.13.0" pyflakes = ">=3.2.0,<3.3.0" [[package]] @@ -3210,13 +3210,13 @@ files = [ [[package]] name = "pycodestyle" -version = "2.11.1" +version = "2.12.0" description = "Python style guide checker" optional = false python-versions = ">=3.8" files = [ - {file = "pycodestyle-2.11.1-py2.py3-none-any.whl", hash = "sha256:44fe31000b2d866f2e41841b18528a505fbd7fef9017b04eff4e2648a0fadc67"}, - {file = "pycodestyle-2.11.1.tar.gz", hash = "sha256:41ba0e7afc9752dfb53ced5489e89f8186be00e599e712660695b7a75ff2663f"}, + {file = "pycodestyle-2.12.0-py2.py3-none-any.whl", hash = "sha256:949a39f6b86c3e1515ba1787c2022131d165a8ad271b11370a8819aa070269e4"}, + {file = "pycodestyle-2.12.0.tar.gz", hash = "sha256:442f950141b4f43df752dd303511ffded3a04c2b6fb7f65980574f0c31e6e79c"}, ] [[package]] @@ -4752,4 +4752,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.12.2" -content-hash = "bf8c577b2f05c2cfbdea05e5006b1685b94981ca8b3156ac9c39d3c8a399a72e" +content-hash = "da984134a9968f2bc665b499dee6236a617cefbe7f35e1162ddd9c7e3c7a622f" diff --git a/pyproject.toml b/pyproject.toml index de04706dc..76b8301c7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,7 +86,7 @@ bandit = "*" black = "^24.3.0" cloudfoundry-client = "*" exceptiongroup = "==1.2.1" -flake8 = "^7.0.0" +flake8 = "^7.1.0" flake8-bugbear = "^24.1.17" freezegun = "^1.5.1" honcho = "*" From c123614cc4892f6417967715d0a16be1c9ec58d5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 28 Jun 2024 21:08:57 +0000 Subject: [PATCH 23/45] Bump botocore from 1.34.134 to 1.34.136 Bumps [botocore](https://github.com/boto/botocore) from 1.34.134 to 1.34.136. - [Changelog](https://github.com/boto/botocore/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/botocore/compare/1.34.134...1.34.136) --- updated-dependencies: - dependency-name: botocore dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- poetry.lock | 16 ++++++++-------- pyproject.toml | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/poetry.lock b/poetry.lock index b25e061bd..9ae169579 100644 --- a/poetry.lock +++ b/poetry.lock @@ -204,17 +204,17 @@ tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "p [[package]] name = "awscli" -version = "1.33.16" +version = "1.33.18" description = "Universal Command Line Environment for AWS." optional = false python-versions = ">=3.8" files = [ - {file = "awscli-1.33.16-py3-none-any.whl", hash = "sha256:8eef82e6c5c3d1f6c881ed4558eaf790c6e433670d4fb5cac622ed5a74d54c98"}, - {file = "awscli-1.33.16.tar.gz", hash = "sha256:5550ca894ab66974061ad3f8fef7a9bc579e5dd8f25a97eebf356e9d8abd5907"}, + {file = "awscli-1.33.18-py3-none-any.whl", hash = "sha256:4065a0c9ee7bd2281e0b04616242693abbe17cd9d7be966abc7a850d5044226d"}, + {file = "awscli-1.33.18.tar.gz", hash = "sha256:800cae2c020dae7e86877e2b53dee637c19acc62de8084bc67e3434ac174ca35"}, ] [package.dependencies] -botocore = "1.34.134" +botocore = "1.34.136" colorama = ">=0.2.5,<0.4.7" docutils = ">=0.10,<0.17" PyYAML = ">=3.10,<6.1" @@ -422,13 +422,13 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.34.134" +version = "1.34.136" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">=3.8" files = [ - {file = "botocore-1.34.134-py3-none-any.whl", hash = "sha256:45219e00639755f92569b29f8f279d5dde721494791412c1f7026a3779e8d9f4"}, - {file = "botocore-1.34.134.tar.gz", hash = "sha256:e29c299599426ed16dd2d4c1e20eef784f96b15e1850ebbc59a3250959285b95"}, + {file = "botocore-1.34.136-py3-none-any.whl", hash = "sha256:c63fe9032091fb9e9477706a3ebfa4d0c109b807907051d892ed574f9b573e61"}, + {file = "botocore-1.34.136.tar.gz", hash = "sha256:7f7135178692b39143c8f152a618d2a3b71065a317569a7102d2306d4946f42f"}, ] [package.dependencies] @@ -4752,4 +4752,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.12.2" -content-hash = "da984134a9968f2bc665b499dee6236a617cefbe7f35e1162ddd9c7e3c7a622f" +content-hash = "8d280f28d1dc6b29c4f9589a903b870e15d265ad3c1b5a0977c39ca3e77940d4" diff --git a/pyproject.toml b/pyproject.toml index 76b8301c7..9e2df57b8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ alembic = "==1.13.2" amqp = "==5.2.0" beautifulsoup4 = "==4.12.3" boto3 = "^1.34.131" -botocore = "^1.34.134" +botocore = "^1.34.136" cachetools = "==5.3.3" celery = {version = "==5.4.0", extras = ["redis"]} certifi = ">=2022.12.7" From df31d43a9264a0242ea54fffdfc0b93025607f56 Mon Sep 17 00:00:00 2001 From: Carlo Costino Date: Fri, 28 Jun 2024 17:22:30 -0400 Subject: [PATCH 24/45] Update pull request template and docs This changeset updates our pull request template to be much more streamlined and shifts most of the information to our documentation. The PR template now links to the docs for folks who are new and unfamiliar with what we require in our pull requests so that the template itself just has the headings and quick outlines to get started more easily and quickly. Signed-off-by: Carlo Costino --- .github/pull_request_template.md | 77 +++----------------- README.md | 8 +++ docs/all.md | 118 ++++++++++++++++++++++++++++--- 3 files changed, 125 insertions(+), 78 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index cb6b2c84b..a87db3fcd 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,79 +1,22 @@ -*A note to PR reviewers: it may be helpful to review our -[code review documentation](https://github.com/GSA/notifications-api/blob/main/docs/all.md#code-reviews) -to know what to keep in mind while reviewing pull requests.* +*A note to PR reviewers: it may be helpful to review our [code review documentation](https://github.com/GSA/notifications-api/blob/main/docs/all.md#code-reviews) to know what to keep in mind while reviewing pull requests.* ## Description -Please enter a clear description about your proposed changes and what the -expected outcome(s) is/are from there. If there are complex implementation -details within the changes, this is a great place to explain those details using -plain language. - -This should include: - -- Links to issues that this PR addresses -- Screenshots or screen captures of any visible changes, especially for UI work -- Dependency changes - -If there are any caveats, known issues, follow-up items, etc., make a quick note -of them here as well, though more details are probably warranted in the issue -itself in this case. +Please enter a detailed description here. ## TODO (optional) -If you're opening a draft PR, it might be helpful to list any outstanding work, -especially if you're asking folks to take a look before it's ready for full -review. In this case, create a small checklist with the outstanding items: - -- [ ] TODO item 1 -- [ ] TODO item 2 -- [ ] TODO item ... +* [ ] TODO item 1 +* [ ] TODO item 2 +* [ ] TODO item ... ## Security Considerations -Please think about the security compliance aspect of your changes and what the -potential impacts might be. - -**NOTE: Please be mindful of sharing sensitive information here! If you're not -sure of what to write, please ask the team first before writing anything here.** - -Relevant details could include (and are not limited to) the following: - -- Handling secrets/credential management (or specifically calling out that there - is nothing to handle) -- Any adjustments to the flow of data in and out the system, or even within it -- Connecting or disconnecting any external services to the application -- Handling of any sensitive information, such as PII -- Handling of information within log statements or other application monitoring - services/hooks -- The inclusion of a new external dependency or the removal of an existing one -- ... (anything else relevant from a security compliance perspective) - -There are some cases where there are no security considerations to be had, e.g., -updating our documentation with publicly available information. In those cases -it is fine to simply put something like this: - -- None; this is a documentation update with publicly available information. +* Consideration 1 +* Consideration 2 +* Consideration ... diff --git a/README.md b/README.md index 8547e540a..0ae5d67c6 100644 --- a/README.md +++ b/README.md @@ -493,6 +493,14 @@ instructions above for more details. - [Celery scheduled tasks](./docs/all.md#celery-scheduled-tasks) - [Notify.gov](./docs/all.md#notifygov) - [System Description](./docs/all.md#system-description) +- [Pull Requests](.docs/all.md#pull-requests) + - [Getting Started](.docs/all.md#getting-started) + - [Description](.docs/all.md#description) + - [TODO (optional)](.docs/all.md#todo-(optional)) + - [Security Considerations](.docs/all.md#security-considerations) +- [Code Reviews](.docs/all.md#code-reviews) + - [For the reviewer](.docs/all.md#for-the-reviewer) + - [For the author](.docs/all.md#for-the-author) - [Run Book](./docs/all.md#run-book) - [ Alerts, Notifications, Monitoring](./docs/all.md#-alerts-notifications-monitoring) - [ Restaging Apps](./docs/all.md#-restaging-apps) diff --git a/docs/all.md b/docs/all.md index 0ad78ae2b..f06f17ff8 100644 --- a/docs/all.md +++ b/docs/all.md @@ -38,6 +38,11 @@ - [Celery scheduled tasks](#celery-scheduled-tasks) - [Notify.gov](#notifygov) - [System Description](#system-description) +- [Pull Requests](#pull-requests) + - [Getting Started](#getting-started) + - [Description](#description) + - [TODO (optional)](#todo-(optional)) + - [Security Considerations](#security-considerations) - [Code Reviews](#code-reviews) - [For the reviewer](#for-the-reviewer) - [For the author](#for-the-author) @@ -817,6 +822,97 @@ Notify.gov also provisions and uses two AWS services via a [supplemental service For further details of the system and how it connects to supporting services, see the [application boundary diagram](https://github.com/GSA/us-notify-compliance/blob/main/diagrams/rendered/apps/application.boundary.png) +Pull Requests +============= + +Changes are made to our applications via pull requests, which show a diff +(the before and after state of all proposed changes in the code) of of the work +done for that particular branch. We use pull requests as the basis for working +on Notify.gov and modifying the application over time for improvements, bug +fixes, new features, and more. + +There are several things that make for a good and complete pull request: + +* An appropriate and descriptive title +* A detailed description of what's being changed, including any outstanding work + (TODOs) +* A list of security considerations, which contains information about anything + we need to be mindful of from a security compliance perspective +* The proper labels, assignee, code reviewer, and other project metadata set + + +### Getting Started + +When you first open a pull request, start off by making sure the metadata for it +is in place: + +* Provide an appropriate and descriptive title for the pull request +* Link the pull request to its corresponding issue (must be done after creating + the pull request itself) +* Assign yourself as the author +* Attach the appropriate labels to it +* Set it to be on the Notify.gov project board +* Select one or more reviewers from the team or mark the pull request as a draft + depending on its current state + * If the pull request is a draft, please be sure to add reviewers once it is + ready for review and mark it ready for review + +### Description + +Please enter a clear description about your proposed changes and what the +expected outcome(s) is/are from there. If there are complex implementation +details within the changes, this is a great place to explain those details using +plain language. + +This should include: + +* Links to issues that this PR addresses (especially if more than one) +* Screenshots or screen captures of any visible changes, especially for UI work +* Dependency changes + +If there are any caveats, known issues, follow-up items, etc., make a quick note +of them here as well, though more details are probably warranted in the issue +itself in this case. + +### TODO (optional) + +If you're opening a draft PR, it might be helpful to list any outstanding work, +especially if you're asking folks to take a look before it's ready for full +review. In this case, create a small checklist with the outstanding items: + +* [ ] TODO item 1 +* [ ] TODO item 2 +* [ ] TODO item ... + +### Security Considerations + +Please think about the security compliance aspect of your changes and what the +potential impacts might be. + +**NOTE: Please be mindful of sharing sensitive information here! If you're not sure of what to write, please ask the team first before writing anything here.** + +Relevant details could include (and are not limited to) the following: + +* Handling secrets/credential management (or specifically calling out that there + is nothing to handle) +* Any adjustments to the flow of data in and out the system, or even within it +* Connecting or disconnecting any external services to the application +* Handling of any sensitive information, such as PII +* Handling of information within log statements or other application monitoring + services/hooks +* The inclusion of a new external dependency or the removal of an existing one +* ... (anything else relevant from a security compliance perspective) + +There are some cases where there are no security considerations to be had, e.g., +updating our documentation with publicly available information. In those cases +it is fine to simply put something like this: + +* None; this is a documentation update with publicly available information. + +This way it shows that we still gave this section consideration and that nothing +happens to apply in this scenario. + + Code Reviews ============ @@ -856,19 +952,19 @@ behavior and lack of professionalism is not acceptable or tolerated.** When performing a code review, it is helpful to keep the following guidelines in mind: -- Be on the lookout for any sensitive information and/or leaked credentials, +* Be on the lookout for any sensitive information and/or leaked credentials, secrets, PII, etc. -- Ask and call out things that aren't clear to you; it never hurts to double +* Ask and call out things that aren't clear to you; it never hurts to double check your understanding of something! -- Check that things are named descriptively and appropriately and call out +* Check that things are named descriptively and appropriately and call out anything that is not. -- Check that comments are present for complex areas when needed. -- Make sure the pull request itself is properly prepared - it has a clear +* Check that comments are present for complex areas when needed. +* Make sure the pull request itself is properly prepared - it has a clear description, calls out security concerns, and has the necessary labels, flags, issue link, etc., set on it. -- Do not be shy about using the suggested changes feature in GitHub pull request +* Do not be shy about using the suggested changes feature in GitHub pull request comments; this can help save a lot of time! -- Do not be shy about marking a review with the `Request Changes` status - yes, +* Do not be shy about marking a review with the `Request Changes` status - yes, it looks big and red when it shows up, but this is completely fine and not to be taken as a personal mark against the author(s) of the pull request! @@ -896,14 +992,14 @@ behavior and lack of professionalism is not acceptable or tolerated.** When going over a review, it may be helpful to keep these perspectives in mind: -- Approach the review with an open mind, curiosity, and appreciation. -- If anything the reviewer(s) mentions is unclear to you, please ask for +* Approach the review with an open mind, curiosity, and appreciation. +* If anything the reviewer(s) mentions is unclear to you, please ask for clarification and engage them in further dialogue! -- If you disagree with a suggestion or request, please say so and engage in an +* If you disagree with a suggestion or request, please say so and engage in an open and respecful dialogue to come to a mutual understanding of what the appropriate next step(S) should be - accept the change, reject the change, take a different path entirely, etc. -- If there are no issues with any suggested edits or requested changes, make +* If there are no issues with any suggested edits or requested changes, make the necessary adjustments and let the reviewer(s) know when the work is ready for review again. From 76742377687fcd11ba0a373c5ba2326f22a88771 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 28 Jun 2024 21:31:10 +0000 Subject: [PATCH 25/45] Bump redis from 5.0.6 to 5.0.7 Bumps [redis](https://github.com/redis/redis-py) from 5.0.6 to 5.0.7. - [Release notes](https://github.com/redis/redis-py/releases) - [Changelog](https://github.com/redis/redis-py/blob/master/CHANGES) - [Commits](https://github.com/redis/redis-py/compare/v5.0.6...v5.0.7) --- updated-dependencies: - dependency-name: redis dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- poetry.lock | 8 ++++---- pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index 9ae169579..b2b8c080e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3635,13 +3635,13 @@ full = ["numpy"] [[package]] name = "redis" -version = "5.0.6" +version = "5.0.7" description = "Python client for Redis database and key-value store" optional = false python-versions = ">=3.7" files = [ - {file = "redis-5.0.6-py3-none-any.whl", hash = "sha256:c0d6d990850c627bbf7be01c5c4cbaadf67b48593e913bb71c9819c30df37eee"}, - {file = "redis-5.0.6.tar.gz", hash = "sha256:38473cd7c6389ad3e44a91f4c3eaf6bcb8a9f746007f29bf4fb20824ff0b2197"}, + {file = "redis-5.0.7-py3-none-any.whl", hash = "sha256:0e479e24da960c690be5d9b96d21f7b918a98c0cf49af3b6fafaa0753f93a0db"}, + {file = "redis-5.0.7.tar.gz", hash = "sha256:8f611490b93c8109b50adc317b31bfd84fff31def3475b92e7e80bf39f48175b"}, ] [package.extras] @@ -4752,4 +4752,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.12.2" -content-hash = "8d280f28d1dc6b29c4f9589a903b870e15d265ad3c1b5a0977c39ca3e77940d4" +content-hash = "64e90b26303ffae001b0aa918e0ac3167a7a1d91a83f4ad926dab26cdd1c20c6" diff --git a/pyproject.toml b/pyproject.toml index 9e2df57b8..befa619b4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,7 +76,7 @@ urllib3 = "^2.2.2" webencodings = "^0.5.1" itsdangerous = "^2.2.0" jinja2 = "^3.1.4" -redis = "^5.0.6" +redis = "^5.0.7" requests = "^2.32.3" From 365fb0ffd9c406fc3e9ea3772970903eeb59f507 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 28 Jun 2024 22:05:07 +0000 Subject: [PATCH 26/45] Bump boto3 from 1.34.131 to 1.34.136 Bumps [boto3](https://github.com/boto/boto3) from 1.34.131 to 1.34.136. - [Release notes](https://github.com/boto/boto3/releases) - [Commits](https://github.com/boto/boto3/compare/1.34.131...1.34.136) --- updated-dependencies: - dependency-name: boto3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- poetry.lock | 10 +++++----- pyproject.toml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/poetry.lock b/poetry.lock index b2b8c080e..94300b1c9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -403,17 +403,17 @@ files = [ [[package]] name = "boto3" -version = "1.34.131" +version = "1.34.136" description = "The AWS SDK for Python" optional = false python-versions = ">=3.8" files = [ - {file = "boto3-1.34.131-py3-none-any.whl", hash = "sha256:05e388cb937e82be70bfd7eb0c84cf8011ff35cf582a593873ac21675268683b"}, - {file = "boto3-1.34.131.tar.gz", hash = "sha256:dab8f72a6c4e62b4fd70da09e08a6b2a65ea2115b27dd63737142005776ef216"}, + {file = "boto3-1.34.136-py3-none-any.whl", hash = "sha256:d41037e2c680ab8d6c61a0a4ee6bf1fdd9e857f43996672830a95d62d6f6fa79"}, + {file = "boto3-1.34.136.tar.gz", hash = "sha256:0314e6598f59ee0f34eb4e6d1a0f69fa65c146d2b88a6e837a527a9956ec2731"}, ] [package.dependencies] -botocore = ">=1.34.131,<1.35.0" +botocore = ">=1.34.136,<1.35.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.10.0,<0.11.0" @@ -4752,4 +4752,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.12.2" -content-hash = "64e90b26303ffae001b0aa918e0ac3167a7a1d91a83f4ad926dab26cdd1c20c6" +content-hash = "d3ca67b44f40fb25b724b8468e07d30901ddced875ffe5d6b6710a17e492b072" diff --git a/pyproject.toml b/pyproject.toml index befa619b4..5d6d8b8df 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ python = "^3.12.2" alembic = "==1.13.2" amqp = "==5.2.0" beautifulsoup4 = "==4.12.3" -boto3 = "^1.34.131" +boto3 = "^1.34.136" botocore = "^1.34.136" cachetools = "==5.3.3" celery = {version = "==5.4.0", extras = ["redis"]} From 33cd4778296f50fa0fdd6568c679b6e55e519d32 Mon Sep 17 00:00:00 2001 From: Carlo Costino Date: Fri, 28 Jun 2024 18:18:19 -0400 Subject: [PATCH 27/45] Attempt to prevent poetry.lock from constantly changing The poetry.lock file gets modified locally when running make bootstrap, and it seems like there were a couple of lingering changes not added to the file. This changeset is an attempt to get it to fully sync properly. Signed-off-by: Carlo Costino --- poetry.lock | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/poetry.lock b/poetry.lock index 94300b1c9..d5bc303bb 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. [[package]] name = "aiohttp" @@ -2098,13 +2098,9 @@ files = [ {file = "lxml-5.2.2-cp36-cp36m-win_amd64.whl", hash = "sha256:edcfa83e03370032a489430215c1e7783128808fd3e2e0a3225deee278585196"}, {file = "lxml-5.2.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:28bf95177400066596cdbcfc933312493799382879da504633d16cf60bba735b"}, {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3a745cc98d504d5bd2c19b10c79c61c7c3df9222629f1b6210c0368177589fb8"}, - {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1b590b39ef90c6b22ec0be925b211298e810b4856909c8ca60d27ffbca6c12e6"}, {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b336b0416828022bfd5a2e3083e7f5ba54b96242159f83c7e3eebaec752f1716"}, - {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_28_aarch64.whl", hash = "sha256:c2faf60c583af0d135e853c86ac2735ce178f0e338a3c7f9ae8f622fd2eb788c"}, {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:4bc6cb140a7a0ad1f7bc37e018d0ed690b7b6520ade518285dc3171f7a117905"}, - {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7ff762670cada8e05b32bf1e4dc50b140790909caa8303cfddc4d702b71ea184"}, {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:57f0a0bbc9868e10ebe874e9f129d2917750adf008fe7b9c1598c0fbbfdde6a6"}, - {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:a6d2092797b388342c1bc932077ad232f914351932353e2e8706851c870bca1f"}, {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:60499fe961b21264e17a471ec296dcbf4365fbea611bf9e303ab69db7159ce61"}, {file = "lxml-5.2.2-cp37-cp37m-win32.whl", hash = "sha256:d9b342c76003c6b9336a80efcc766748a333573abf9350f4094ee46b006ec18f"}, {file = "lxml-5.2.2-cp37-cp37m-win_amd64.whl", hash = "sha256:b16db2770517b8799c79aa80f4053cd6f8b716f21f8aca962725a9565ce3ee40"}, @@ -3475,7 +3471,6 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, From 5c305177a081ec5e1110024503d6c9ae2a9e0640 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Mon, 1 Jul 2024 10:35:20 -0700 Subject: [PATCH 28/45] add more debug and add command to download csv files --- app/commands.py | 11 ++++++++ notifications_utils/recipients.py | 28 ++++++++++++++++++- poetry.lock | 8 ++---- .../test_send_notification.py | 2 +- tests/app/user/test_rest.py | 4 +-- .../test_recipient_validation.py | 6 ++++ 6 files changed, 49 insertions(+), 10 deletions(-) diff --git a/app/commands.py b/app/commands.py index 826c2013b..898ecdb89 100644 --- a/app/commands.py +++ b/app/commands.py @@ -589,6 +589,17 @@ def process_row_from_job(job_id, job_row_number): ) +@notify_command(name="download-csv-file-by-name") +@click.option("-f", "--csv_filename", required=True, help="csv file name") +def download_csv_file_by_name(csv_filename): + + bucket_name = getenv("CSV_BUCKET_NAME") + access_key = getenv("CSV_AWS_ACCESS_KEY_ID") + secret = getenv("CSV_AWS_SECRET_ACCESS_KEY") + region = getenv("CSV_AWS_REGION") + print(s3.get_s3_file(bucket_name, csv_filename, access_key, secret, region)) + + @notify_command(name="populate-annual-billing-with-the-previous-years-allowance") @click.option( "-y", diff --git a/notifications_utils/recipients.py b/notifications_utils/recipients.py index 68e2cb101..0e40ea2ab 100644 --- a/notifications_utils/recipients.py +++ b/notifications_utils/recipients.py @@ -602,12 +602,34 @@ def validate_us_phone_number(number): raise InvalidPhoneError(exc._msg) from exc +def show_mangled_number_clues(number): + print(f"ORIG {number}") + translator = { + "1": "X", + "2": "X", + "3": "X", + "4": "X", + "5": "X", + "6": "X", + "7": "X", + "8": "X", + "9": "X", + "0": "X", + } + for key in translator: + number = number.replace(key, translator[key]) + + return number + + def validate_phone_number(number, international=False): if (not international) or is_us_phone_number(number): + print("FAST EXIT") return validate_us_phone_number(number) try: parsed = phonenumbers.parse(number, None) + print("PARSED") if parsed.country_code != 1: raise InvalidPhoneError("Invalid country code") number = f"{parsed.country_code}{parsed.national_number}" @@ -619,7 +641,11 @@ def validate_phone_number(number, international=False): except NumberParseException as exc: if exc._msg == "Could not interpret numbers after plus-sign.": raise InvalidPhoneError("Not a valid country prefix") from exc - raise InvalidPhoneError(exc._msg) from exc + if not isinstance(number, str): + raise InvalidPhoneError(f"Number must be string, not type {type(number)}") + raise InvalidPhoneError( + f"Invalid phone number looks like {show_mangled_number_clues(number)} {exc._msg}" + ) validate_and_format_phone_number = validate_phone_number diff --git a/poetry.lock b/poetry.lock index 1c5be2257..28d284d0b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. [[package]] name = "aiohttp" @@ -2098,13 +2098,9 @@ files = [ {file = "lxml-5.2.2-cp36-cp36m-win_amd64.whl", hash = "sha256:edcfa83e03370032a489430215c1e7783128808fd3e2e0a3225deee278585196"}, {file = "lxml-5.2.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:28bf95177400066596cdbcfc933312493799382879da504633d16cf60bba735b"}, {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3a745cc98d504d5bd2c19b10c79c61c7c3df9222629f1b6210c0368177589fb8"}, - {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1b590b39ef90c6b22ec0be925b211298e810b4856909c8ca60d27ffbca6c12e6"}, {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b336b0416828022bfd5a2e3083e7f5ba54b96242159f83c7e3eebaec752f1716"}, - {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_28_aarch64.whl", hash = "sha256:c2faf60c583af0d135e853c86ac2735ce178f0e338a3c7f9ae8f622fd2eb788c"}, {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:4bc6cb140a7a0ad1f7bc37e018d0ed690b7b6520ade518285dc3171f7a117905"}, - {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7ff762670cada8e05b32bf1e4dc50b140790909caa8303cfddc4d702b71ea184"}, {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:57f0a0bbc9868e10ebe874e9f129d2917750adf008fe7b9c1598c0fbbfdde6a6"}, - {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:a6d2092797b388342c1bc932077ad232f914351932353e2e8706851c870bca1f"}, {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:60499fe961b21264e17a471ec296dcbf4365fbea611bf9e303ab69db7159ce61"}, {file = "lxml-5.2.2-cp37-cp37m-win32.whl", hash = "sha256:d9b342c76003c6b9336a80efcc766748a333573abf9350f4094ee46b006ec18f"}, {file = "lxml-5.2.2-cp37-cp37m-win_amd64.whl", hash = "sha256:b16db2770517b8799c79aa80f4053cd6f8b716f21f8aca962725a9565ce3ee40"}, @@ -2493,6 +2489,7 @@ files = [ {file = "msgpack-1.0.8-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5fbb160554e319f7b22ecf530a80a3ff496d38e8e07ae763b9e82fadfe96f273"}, {file = "msgpack-1.0.8-cp39-cp39-win32.whl", hash = "sha256:f9af38a89b6a5c04b7d18c492c8ccf2aee7048aff1ce8437c4683bb5a1df893d"}, {file = "msgpack-1.0.8-cp39-cp39-win_amd64.whl", hash = "sha256:ed59dd52075f8fc91da6053b12e8c89e37aa043f8986efd89e61fae69dc1b011"}, + {file = "msgpack-1.0.8-py3-none-any.whl", hash = "sha256:24f727df1e20b9876fa6e95f840a2a2651e34c0ad147676356f4bf5fbb0206ca"}, {file = "msgpack-1.0.8.tar.gz", hash = "sha256:95c02b0e27e706e48d0e5426d1710ca78e0f0628d6e89d5b5a5b91a5f12274f3"}, ] @@ -3475,7 +3472,6 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, diff --git a/tests/app/service/send_notification/test_send_notification.py b/tests/app/service/send_notification/test_send_notification.py index 9b833bfd0..036c5bac8 100644 --- a/tests/app/service/send_notification/test_send_notification.py +++ b/tests/app/service/send_notification/test_send_notification.py @@ -79,7 +79,7 @@ def test_should_reject_bad_phone_numbers(notify_api, sample_template, mocker): assert json_resp["result"] == "error" assert len(json_resp["message"].keys()) == 1 assert ( - "Invalid phone number: The string supplied did not seem to be a phone number." + "Invalid phone number: Invalid phone number looks like invalid The string supplied did not seem to be a phone number." # noqa in json_resp["message"]["to"] ) assert response.status_code == 400 diff --git a/tests/app/user/test_rest.py b/tests/app/user/test_rest.py index a388d264e..dc7b78e82 100644 --- a/tests/app/user/test_rest.py +++ b/tests/app/user/test_rest.py @@ -226,7 +226,7 @@ def test_cannot_create_user_with_empty_strings(admin_request, notify_db_session) assert resp["message"] == { "email_address": ["Not a valid email address"], "mobile_number": [ - "Invalid phone number: The string supplied did not seem to be a phone number." + "Invalid phone number: Invalid phone number looks like The string supplied did not seem to be a phone number." # noqa ], "name": ["Invalid name"], } @@ -949,7 +949,7 @@ def test_cannot_update_user_with_mobile_number_as_empty_string( _expected_status=400, ) assert resp["message"]["mobile_number"] == [ - "Invalid phone number: The string supplied did not seem to be a phone number." + "Invalid phone number: Invalid phone number looks like The string supplied did not seem to be a phone number." # noqa ] diff --git a/tests/notifications_utils/test_recipient_validation.py b/tests/notifications_utils/test_recipient_validation.py index ff48df775..cc6c2a676 100644 --- a/tests/notifications_utils/test_recipient_validation.py +++ b/tests/notifications_utils/test_recipient_validation.py @@ -9,6 +9,7 @@ from notifications_utils.recipients import ( get_international_phone_info, international_phone_info, is_us_phone_number, + show_mangled_number_clues, try_validate_and_format_phone_number, validate_and_format_phone_number, validate_email_address, @@ -324,6 +325,11 @@ def test_phone_number_rejects_invalid_international_values(phone_number, error_m assert error_message == str(e.value) +def test_show_mangled_number_clues(): + x = show_mangled_number_clues("848!!-202?-2020$$") + assert x == "XXX!!-XXX?-XXXX$$" + + @pytest.mark.parametrize("email_address", valid_email_addresses) def test_validate_email_address_accepts_valid(email_address): try: From 9a4dd043ce77fdce79cc7853d86c0cbb8d96a78a Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Mon, 1 Jul 2024 10:52:03 -0700 Subject: [PATCH 29/45] cleanup --- docs/all.md | 25 +++++++++++++++++++++++++ notifications_utils/recipients.py | 2 -- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/docs/all.md b/docs/all.md index 0ad78ae2b..6ee0fbbf7 100644 --- a/docs/all.md +++ b/docs/all.md @@ -55,6 +55,9 @@ - [Data Storage Policies \& Procedures](#data-storage-policies--procedures) - [Potential PII Locations](#potential-pii-locations) - [Data Retention Policy](#data-retention-policy) +- [Debug messages not being sent](#debug-messages-not-being-sent) + - [Getting the file location and tracing what happens](#getting-the-file-location-and-tracing-what-happens) + - [Viewing the csv file](#viewing-the-csv-file) # Infrastructure overview @@ -1224,3 +1227,25 @@ Data Retention Policy Seven (7) days by default. Each service can be set with a custom policy via `ServiceDataRetention` by a Platform Admin. The `ServiceDataRetention` setting applies per-service and per-message type and controls both entries in the `notifications` table as well as `csv` contact files uploaded to s3 Data cleanup is controlled by several tasks in the `nightly_tasks.py` file, kicked off by Celery Beat. + + +# Debug messages not being sent + + +## Getting the file location and tracing what happens + + +Ask the user to provide the csv file name. Either the csv file they uploaded, or the one that is autogenerated when they do a one-off send and is visible in the UI + +Starting with the admin logs, search for this file name. When you find it, the log line should have the file name linked to the job_id and the csv file location. Save both of these. + +In the api logs, search by job_id. Either you will see evidence of the job failing and retrying over and over (in which case search for a stack trace using timestamp), or you will ultimately get to a log line that links the job_id to a message_id. In this case, now search by message_id. You should be able to find the actual result from AWS, either success or failure, with hopefully some helpful info. + +## Viewing the csv file + +If you need to view th questionable csv file, run the following command: + + +``` +cf run-task notify-api "flask command download_csv_file_by_name -f " +``` diff --git a/notifications_utils/recipients.py b/notifications_utils/recipients.py index 0e40ea2ab..b147dd52b 100644 --- a/notifications_utils/recipients.py +++ b/notifications_utils/recipients.py @@ -624,12 +624,10 @@ def show_mangled_number_clues(number): def validate_phone_number(number, international=False): if (not international) or is_us_phone_number(number): - print("FAST EXIT") return validate_us_phone_number(number) try: parsed = phonenumbers.parse(number, None) - print("PARSED") if parsed.country_code != 1: raise InvalidPhoneError("Invalid country code") number = f"{parsed.country_code}{parsed.national_number}" From 8a3be92a09bcacd92550145b27fdbe2019866a71 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Mon, 1 Jul 2024 11:53:29 -0700 Subject: [PATCH 30/45] fix code inspection --- notifications_utils/recipients.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notifications_utils/recipients.py b/notifications_utils/recipients.py index b147dd52b..34cd2def2 100644 --- a/notifications_utils/recipients.py +++ b/notifications_utils/recipients.py @@ -603,7 +603,7 @@ def validate_us_phone_number(number): def show_mangled_number_clues(number): - print(f"ORIG {number}") + translator = { "1": "X", "2": "X", From 9cacb9cac97b590eee917b18744d5f4349988036 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Jul 2024 21:39:32 +0000 Subject: [PATCH 31/45] Bump moto from 5.0.9 to 5.0.10 Bumps [moto](https://github.com/getmoto/moto) from 5.0.9 to 5.0.10. - [Release notes](https://github.com/getmoto/moto/releases) - [Changelog](https://github.com/getmoto/moto/blob/master/CHANGELOG.md) - [Commits](https://github.com/getmoto/moto/compare/5.0.9...5.0.10) --- updated-dependencies: - dependency-name: moto dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- poetry.lock | 15 ++++++++++----- pyproject.toml | 2 +- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/poetry.lock b/poetry.lock index d5bc303bb..53bbee575 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. [[package]] name = "aiohttp" @@ -2098,9 +2098,13 @@ files = [ {file = "lxml-5.2.2-cp36-cp36m-win_amd64.whl", hash = "sha256:edcfa83e03370032a489430215c1e7783128808fd3e2e0a3225deee278585196"}, {file = "lxml-5.2.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:28bf95177400066596cdbcfc933312493799382879da504633d16cf60bba735b"}, {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3a745cc98d504d5bd2c19b10c79c61c7c3df9222629f1b6210c0368177589fb8"}, + {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1b590b39ef90c6b22ec0be925b211298e810b4856909c8ca60d27ffbca6c12e6"}, {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b336b0416828022bfd5a2e3083e7f5ba54b96242159f83c7e3eebaec752f1716"}, + {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_28_aarch64.whl", hash = "sha256:c2faf60c583af0d135e853c86ac2735ce178f0e338a3c7f9ae8f622fd2eb788c"}, {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:4bc6cb140a7a0ad1f7bc37e018d0ed690b7b6520ade518285dc3171f7a117905"}, + {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7ff762670cada8e05b32bf1e4dc50b140790909caa8303cfddc4d702b71ea184"}, {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:57f0a0bbc9868e10ebe874e9f129d2917750adf008fe7b9c1598c0fbbfdde6a6"}, + {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:a6d2092797b388342c1bc932077ad232f914351932353e2e8706851c870bca1f"}, {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:60499fe961b21264e17a471ec296dcbf4365fbea611bf9e303ab69db7159ce61"}, {file = "lxml-5.2.2-cp37-cp37m-win32.whl", hash = "sha256:d9b342c76003c6b9336a80efcc766748a333573abf9350f4094ee46b006ec18f"}, {file = "lxml-5.2.2-cp37-cp37m-win_amd64.whl", hash = "sha256:b16db2770517b8799c79aa80f4053cd6f8b716f21f8aca962725a9565ce3ee40"}, @@ -2385,13 +2389,13 @@ files = [ [[package]] name = "moto" -version = "5.0.9" +version = "5.0.10" description = "" optional = false python-versions = ">=3.8" files = [ - {file = "moto-5.0.9-py2.py3-none-any.whl", hash = "sha256:21a13e02f83d6a18cfcd99949c96abb2e889f4bd51c4c6a3ecc8b78765cb854e"}, - {file = "moto-5.0.9.tar.gz", hash = "sha256:eb71f1cba01c70fff1f16086acb24d6d9aeb32830d646d8989f98a29aeae24ba"}, + {file = "moto-5.0.10-py2.py3-none-any.whl", hash = "sha256:9ffae2f64cc8fe95b9a12d63ae7268a7d6bea9993b922905b5abd8197d852cd0"}, + {file = "moto-5.0.10.tar.gz", hash = "sha256:eff37363221c93ea44f95721ae0ddb56f977fe70437a041b6cc641ee90266279"}, ] [package.dependencies] @@ -3471,6 +3475,7 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, @@ -4747,4 +4752,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.12.2" -content-hash = "d3ca67b44f40fb25b724b8468e07d30901ddced875ffe5d6b6710a17e492b072" +content-hash = "a961afb913c7ef7c4b4e67e686ca386efec4958d7d76909f4fe91cc169a37b20" diff --git a/pyproject.toml b/pyproject.toml index 5d6d8b8df..6d73ead53 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -92,7 +92,7 @@ freezegun = "^1.5.1" honcho = "*" isort = "^5.13.2" jinja2-cli = {version = "==0.8.2", extras = ["yaml"]} -moto = "==5.0.9" +moto = "==5.0.10" pip-audit = "*" pre-commit = "^3.7.1" pytest = "^8.2.2" From 141dd51b52b0accf995ede4064ce8420e58109dd Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Tue, 2 Jul 2024 08:50:48 -0700 Subject: [PATCH 32/45] code review feedback --- app/commands.py | 17 +++++++++-------- docs/all.md | 2 +- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/app/commands.py b/app/commands.py index 898ecdb89..d8b9d283e 100644 --- a/app/commands.py +++ b/app/commands.py @@ -593,10 +593,10 @@ def process_row_from_job(job_id, job_row_number): @click.option("-f", "--csv_filename", required=True, help="csv file name") def download_csv_file_by_name(csv_filename): - bucket_name = getenv("CSV_BUCKET_NAME") - access_key = getenv("CSV_AWS_ACCESS_KEY_ID") - secret = getenv("CSV_AWS_SECRET_ACCESS_KEY") - region = getenv("CSV_AWS_REGION") + bucket_name = (current_app.config["CSV_UPLOAD_BUCKET"]["bucket"],) + access_key = (current_app.config["CSV_UPLOAD_BUCKET"]["access_key_id"],) + secret = (current_app.config["CSV_UPLOAD_BUCKET"]["secret_access_key"],) + region = (current_app.config["CSV_UPLOAD_BUCKET"]["region"],) print(s3.get_s3_file(bucket_name, csv_filename, access_key, secret, region)) @@ -865,10 +865,11 @@ def promote_user_to_platform_admin(user_email_address): @notify_command(name="purge-csv-bucket") def purge_csv_bucket(): - bucket_name = getenv("CSV_BUCKET_NAME") - access_key = getenv("CSV_AWS_ACCESS_KEY_ID") - secret = getenv("CSV_AWS_SECRET_ACCESS_KEY") - region = getenv("CSV_AWS_REGION") + bucket_name = (current_app.config["CSV_UPLOAD_BUCKET"]["bucket"],) + access_key = (current_app.config["CSV_UPLOAD_BUCKET"]["access_key_id"],) + secret = (current_app.config["CSV_UPLOAD_BUCKET"]["secret_access_key"],) + region = (current_app.config["CSV_UPLOAD_BUCKET"]["region"],) + print("ABOUT TO RUN PURGE CSV BUCKET") s3.purge_bucket(bucket_name, access_key, secret, region) print("RAN PURGE CSV BUCKET") diff --git a/docs/all.md b/docs/all.md index 6ee0fbbf7..23d378ef5 100644 --- a/docs/all.md +++ b/docs/all.md @@ -1243,7 +1243,7 @@ In the api logs, search by job_id. Either you will see evidence of the job fail ## Viewing the csv file -If you need to view th questionable csv file, run the following command: +If you need to view the questionable csv file, run the following command: ``` From 328c211eb62bae1a5e56912451ca8ff50987f99e Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Tue, 2 Jul 2024 11:55:19 -0700 Subject: [PATCH 33/45] remove task --- app/config.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/app/config.py b/app/config.py index 637ece32f..8d913bdd8 100644 --- a/app/config.py +++ b/app/config.py @@ -199,11 +199,6 @@ class Config(object): "schedule": timedelta(minutes=66), "options": {"queue": QueueNames.PERIODIC}, }, - "report-all-users": { - "task": "report-all-users", - "schedule": timedelta(minutes=2), - "options": {"queue": QueueNames.PERIODIC}, - }, "check-job-status": { "task": "check-job-status", "schedule": crontab(), From f95d3e0b99c930f9047405f5dc7da2a71d20c40b Mon Sep 17 00:00:00 2001 From: Cliff Hill Date: Tue, 2 Jul 2024 15:38:22 -0400 Subject: [PATCH 34/45] Made changes I requested to be done. Signed-off-by: Cliff Hill --- app/dao/date_util.py | 2 +- app/dao/services_dao.py | 29 ++++++++++- app/service/rest.py | 57 ++------------------- tests/app/service/test_statistics_rest.py | 60 ----------------------- 4 files changed, 34 insertions(+), 114 deletions(-) diff --git a/app/dao/date_util.py b/app/dao/date_util.py index cac09dee2..d93986b45 100644 --- a/app/dao/date_util.py +++ b/app/dao/date_util.py @@ -92,4 +92,4 @@ def generate_date_range(start_date, end_date=None, days=0): pass current_date += timedelta(days=1) else: - return "A start_date or number of days must be specified" + return "An end_date or number of days must be specified" diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 74f8094a8..a7c41a28e 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -1,3 +1,4 @@ +from re import I import uuid from datetime import timedelta @@ -8,7 +9,7 @@ from sqlalchemy.sql.expression import and_, asc, case, func from app import db from app.dao.dao_utils import VersionOptions, autocommit, version_class -from app.dao.date_util import get_current_calendar_year +from app.dao.date_util import generate_date_range, get_current_calendar_year from app.dao.organization_dao import dao_get_organization_by_email_address from app.dao.service_sms_sender_dao import insert_service_sms_sender from app.dao.service_user_dao import dao_get_service_user @@ -41,6 +42,7 @@ from app.models import ( User, VerifyCode, ) +from app.service import statistics from app.utils import ( escape_special_characters, get_archived_db_column_value, @@ -689,3 +691,28 @@ def fetch_notification_stats_for_service_by_month_by_user( ) .all() ) + + +def get_specific_days_stats(results, start_date, days=None, end_date=None): + if days is not None and end_date is not None: + raise ValueError("Only set days OR set end_date, not both.") + elif days is not None: + gen_range = generate_date_range(start_date, days=days) + elif end_date is not None: + gen_range = generate_date_range(start_date, end_date) + else: + raise ValueError("Either days or end_date must be set.") + + grouped_results = { + date: [] for date in gen_range + } | { + day.date(): [notification_type, status, day, count] + for notification_type, status, day, count in results + } + + stats = { + day.strftime("%Y-%m-%d"): statistics.format_statistics(rows) + for day, rows in grouped_results.items() + } + + return stats diff --git a/app/service/rest.py b/app/service/rest.py index ea59387d8..516a22c3e 100644 --- a/app/service/rest.py +++ b/app/service/rest.py @@ -78,6 +78,7 @@ from app.dao.services_dao import ( dao_update_service, fetch_notification_stats_for_service_by_month_by_user, get_services_by_partial_name, + get_specific_days_stats, ) from app.dao.templates_dao import dao_get_template_by_id from app.dao.users_dao import get_user_by_id @@ -234,18 +235,7 @@ def get_service_statistics_for_specific_days(service_id, start, days=1): results = dao_fetch_stats_for_service_from_days(service_id, start_date, end_date) - grouped_results = defaultdict(list) - for row in results: - notification_type, status, day, count = row - grouped_results[day.date()].append(row) - - for date in generate_date_range(start_date, days=days): - if date not in grouped_results: - grouped_results[date] = [] - - stats = {} - for day, rows in grouped_results.items(): - stats[day.strftime("%Y-%m-%d")] = statistics.format_statistics(rows) + stats = get_specific_days_stats(results, start_date, days=days) return stats @@ -276,20 +266,7 @@ def get_service_statistics_for_specific_days_by_user( service_id, start_date, end_date, user_id ) - grouped_results = defaultdict(list) - for row in results: - notification_type, status, day, count = row - grouped_results[day.date()].append(row) - - print(grouped_results) - - for date in generate_date_range(start_date, days=days): - if date not in grouped_results: - grouped_results[date] = [] - - stats = {} - for day, rows in grouped_results.items(): - stats[day.strftime("%Y-%m-%d")] = statistics.format_statistics(rows) + stats = get_specific_days_stats(results, start_date, days=days) return stats @@ -744,19 +721,7 @@ def get_single_month_notification_stats_by_user(service_id, user_id): service_id, start_date, end_date, user_id ) - grouped_results = defaultdict(list) - for row in results: - notification_type, status, day, count = row - grouped_results[day.date()].append(row) - - for date in generate_date_range(start_date, end_date): - if date not in grouped_results: - grouped_results[date] = [] - - stats = {} - for day, rows in grouped_results.items(): - stats[day.strftime("%Y-%m-%d")] = statistics.format_statistics(rows) - + stats = get_specific_days_stats(results, start_date, end_date=end_date) return jsonify(stats) @@ -778,19 +743,7 @@ def get_single_month_notification_stats_for_service(service_id): results = dao_fetch_stats_for_service_from_days(service_id, start_date, end_date) - grouped_results = defaultdict(list) - for row in results: - notification_type, status, day, count = row - grouped_results[day.date()].append(row) - - for date in generate_date_range(start_date, end_date): - if date not in grouped_results: - grouped_results[date] = [] - - stats = {} - for day, rows in grouped_results.items(): - stats[day.strftime("%Y-%m-%d")] = statistics.format_statistics(rows) - + stats = get_specific_days_stats(results, start_date, end_date=end_date) return jsonify(stats) diff --git a/tests/app/service/test_statistics_rest.py b/tests/app/service/test_statistics_rest.py index 9769a678e..6d20cacc3 100644 --- a/tests/app/service/test_statistics_rest.py +++ b/tests/app/service/test_statistics_rest.py @@ -294,66 +294,6 @@ def test_get_monthly_notification_stats_returns_stats(admin_request, sample_serv } -# Test removed because new endpoint uses the view which combines this data -# @freeze_time("2016-06-05 12:00:00") -# def test_get_monthly_notification_stats_combines_todays_data_and_historic_stats( -# admin_request, sample_template -# ): -# create_ft_notification_status( -# datetime(2016, 5, 1, 12), -# template=sample_template, -# count=1, -# ) -# create_ft_notification_status( -# datetime(2016, 6, 1, 12), -# template=sample_template, -# notification_status=NotificationStatus.CREATED, -# count=2, -# ) # noqa - -# create_notification( -# sample_template, -# created_at=datetime(2016, 6, 5, 12), -# status=NotificationStatus.CREATED, -# ) -# create_notification( -# sample_template, -# created_at=datetime(2016, 6, 5, 12), -# status=NotificationStatus.DELIVERED, -# ) - -# # this doesn't get returned in the stats because it is old - it should be in ft_notification_status by now -# create_notification( -# sample_template, -# created_at=datetime(2016, 6, 4, 12), -# status=NotificationStatus.SENDING, -# ) - -# response = admin_request.get( -# "service.get_monthly_notification_stats", -# service_id=sample_template.service_id, -# year=2016, -# ) - -# assert len(response["data"]) == 6 # January to June -# assert response["data"]["2016-05"] == { -# NotificationType.SMS: { -# NotificationStatus.DELIVERED: 1, -# StatisticsType.REQUESTED: 1, -# }, -# NotificationType.EMAIL: {}, -# } -# assert response["data"]["2016-06"] == { -# NotificationType.SMS: { -# # combines the stats from the historic ft_notification_status and the current notifications -# NotificationStatus.CREATED: 3, -# NotificationStatus.DELIVERED: 1, -# StatisticsType.REQUESTED: 4, -# }, -# NotificationType.EMAIL: {}, -# } - - def test_get_monthly_notification_stats_ignores_test_keys( admin_request, sample_service ): From 8708d8ddce33976f2d6fd0f9bd8470b3727d252b Mon Sep 17 00:00:00 2001 From: Cliff Hill Date: Tue, 2 Jul 2024 15:57:14 -0400 Subject: [PATCH 35/45] Fixin' imports. Signed-off-by: Cliff Hill --- app/dao/services_dao.py | 1 - app/service/rest.py | 2 -- 2 files changed, 3 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index a7c41a28e..1f889c6b3 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -1,4 +1,3 @@ -from re import I import uuid from datetime import timedelta diff --git a/app/service/rest.py b/app/service/rest.py index 516a22c3e..bef4cc896 100644 --- a/app/service/rest.py +++ b/app/service/rest.py @@ -1,5 +1,4 @@ import itertools -from collections import defaultdict from datetime import datetime, timedelta from flask import Blueprint, current_app, jsonify, request @@ -19,7 +18,6 @@ from app.dao.api_key_dao import ( ) from app.dao.dao_utils import dao_rollback, transaction from app.dao.date_util import ( - generate_date_range, get_calendar_year, get_month_start_and_end_date_in_utc, ) From c368d3d3f2b204c5036e00aded0784e7779b16d9 Mon Sep 17 00:00:00 2001 From: Cliff Hill Date: Tue, 2 Jul 2024 16:00:58 -0400 Subject: [PATCH 36/45] More import adjustments. Signed-off-by: Cliff Hill --- app/dao/services_dao.py | 4 +--- app/service/rest.py | 5 +---- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 1f889c6b3..6fa663341 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -702,9 +702,7 @@ def get_specific_days_stats(results, start_date, days=None, end_date=None): else: raise ValueError("Either days or end_date must be set.") - grouped_results = { - date: [] for date in gen_range - } | { + grouped_results = {date: [] for date in gen_range} | { day.date(): [notification_type, status, day, count] for notification_type, status, day, count in results } diff --git a/app/service/rest.py b/app/service/rest.py index bef4cc896..71faab7a1 100644 --- a/app/service/rest.py +++ b/app/service/rest.py @@ -17,10 +17,7 @@ from app.dao.api_key_dao import ( save_model_api_key, ) from app.dao.dao_utils import dao_rollback, transaction -from app.dao.date_util import ( - get_calendar_year, - get_month_start_and_end_date_in_utc, -) +from app.dao.date_util import get_calendar_year, get_month_start_and_end_date_in_utc from app.dao.fact_notification_status_dao import ( fetch_monthly_template_usage_for_service, fetch_notification_status_for_service_by_month, From fe45366bb0937c62042c419ef745d1c1135222de Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 2 Jul 2024 21:00:47 +0000 Subject: [PATCH 37/45] Bump setuptools from 70.1.1 to 70.2.0 Bumps [setuptools](https://github.com/pypa/setuptools) from 70.1.1 to 70.2.0. - [Release notes](https://github.com/pypa/setuptools/releases) - [Changelog](https://github.com/pypa/setuptools/blob/main/NEWS.rst) - [Commits](https://github.com/pypa/setuptools/compare/v70.1.1...v70.2.0) --- updated-dependencies: - dependency-name: setuptools dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- poetry.lock | 13 ++++++------- pyproject.toml | 2 +- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/poetry.lock b/poetry.lock index cf976e2cd..efe2f8898 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2493,7 +2493,6 @@ files = [ {file = "msgpack-1.0.8-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5fbb160554e319f7b22ecf530a80a3ff496d38e8e07ae763b9e82fadfe96f273"}, {file = "msgpack-1.0.8-cp39-cp39-win32.whl", hash = "sha256:f9af38a89b6a5c04b7d18c492c8ccf2aee7048aff1ce8437c4683bb5a1df893d"}, {file = "msgpack-1.0.8-cp39-cp39-win_amd64.whl", hash = "sha256:ed59dd52075f8fc91da6053b12e8c89e37aa043f8986efd89e61fae69dc1b011"}, - {file = "msgpack-1.0.8-py3-none-any.whl", hash = "sha256:24f727df1e20b9876fa6e95f840a2a2651e34c0ad147676356f4bf5fbb0206ca"}, {file = "msgpack-1.0.8.tar.gz", hash = "sha256:95c02b0e27e706e48d0e5426d1710ca78e0f0628d6e89d5b5a5b91a5f12274f3"}, ] @@ -4022,18 +4021,18 @@ jeepney = ">=0.6" [[package]] name = "setuptools" -version = "70.1.1" +version = "70.2.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-70.1.1-py3-none-any.whl", hash = "sha256:a58a8fde0541dab0419750bcc521fbdf8585f6e5cb41909df3a472ef7b81ca95"}, - {file = "setuptools-70.1.1.tar.gz", hash = "sha256:937a48c7cdb7a21eb53cd7f9b59e525503aa8abaf3584c730dc5f7a5bec3a650"}, + {file = "setuptools-70.2.0-py3-none-any.whl", hash = "sha256:b8b8060bb426838fbe942479c90296ce976249451118ef566a5a0b7d8b78fb05"}, + {file = "setuptools-70.2.0.tar.gz", hash = "sha256:bd63e505105011b25c3c11f753f7e3b8465ea739efddaccef8f0efac2137bac1"}, ] [package.extras] -docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test", "mypy (==1.10.0)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.1)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.3.2)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test", "mypy (==1.10.0)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.3.2)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] [[package]] name = "shapely" @@ -4753,4 +4752,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.12.2" -content-hash = "a961afb913c7ef7c4b4e67e686ca386efec4958d7d76909f4fe91cc169a37b20" +content-hash = "58355cc6de094c921f27347d74c9fbb4867ae2941b4d545e9ac329cf0c5c2bbc" diff --git a/pyproject.toml b/pyproject.toml index 6d73ead53..0bdb1f012 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -102,7 +102,7 @@ pytest-cov = "^5.0.0" pytest-xdist = "^3.5.0" radon = "^6.0.1" requests-mock = "^1.11.0" -setuptools = "^70.1.1" +setuptools = "^70.2.0" sqlalchemy-utils = "^0.41.2" vulture = "^2.10" detect-secrets = "^1.5.0" From 872b4b195c1e9d763cda609ca0560de7b69d8492 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 2 Jul 2024 21:22:50 +0000 Subject: [PATCH 38/45] Bump botocore from 1.34.136 to 1.34.138 Bumps [botocore](https://github.com/boto/botocore) from 1.34.136 to 1.34.138. - [Changelog](https://github.com/boto/botocore/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/botocore/compare/1.34.136...1.34.138) --- updated-dependencies: - dependency-name: botocore dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- poetry.lock | 16 ++++++++-------- pyproject.toml | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/poetry.lock b/poetry.lock index efe2f8898..9c5c4d5e7 100644 --- a/poetry.lock +++ b/poetry.lock @@ -204,17 +204,17 @@ tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "p [[package]] name = "awscli" -version = "1.33.18" +version = "1.33.20" description = "Universal Command Line Environment for AWS." optional = false python-versions = ">=3.8" files = [ - {file = "awscli-1.33.18-py3-none-any.whl", hash = "sha256:4065a0c9ee7bd2281e0b04616242693abbe17cd9d7be966abc7a850d5044226d"}, - {file = "awscli-1.33.18.tar.gz", hash = "sha256:800cae2c020dae7e86877e2b53dee637c19acc62de8084bc67e3434ac174ca35"}, + {file = "awscli-1.33.20-py3-none-any.whl", hash = "sha256:bcb0d8e10ddeb966a2d87b7e5b564c9d849ada0a0819e1fd14e6afc9e319abf9"}, + {file = "awscli-1.33.20.tar.gz", hash = "sha256:d8167ac869b1e3750790944c56d5fd2f468d75163c3d0cbcf78444158541d9b5"}, ] [package.dependencies] -botocore = "1.34.136" +botocore = "1.34.138" colorama = ">=0.2.5,<0.4.7" docutils = ">=0.10,<0.17" PyYAML = ">=3.10,<6.1" @@ -422,13 +422,13 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.34.136" +version = "1.34.138" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">=3.8" files = [ - {file = "botocore-1.34.136-py3-none-any.whl", hash = "sha256:c63fe9032091fb9e9477706a3ebfa4d0c109b807907051d892ed574f9b573e61"}, - {file = "botocore-1.34.136.tar.gz", hash = "sha256:7f7135178692b39143c8f152a618d2a3b71065a317569a7102d2306d4946f42f"}, + {file = "botocore-1.34.138-py3-none-any.whl", hash = "sha256:84e96a954c39a6f09cae4ea95b2ae582b5ae01b5040c92507b60509c9be5377a"}, + {file = "botocore-1.34.138.tar.gz", hash = "sha256:f558bbea96c4a4abbaeeedc477dabb00902311ba1ca6327974a6819b9f384920"}, ] [package.dependencies] @@ -4752,4 +4752,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.12.2" -content-hash = "58355cc6de094c921f27347d74c9fbb4867ae2941b4d545e9ac329cf0c5c2bbc" +content-hash = "5ac3c87860d9d9ccce2ed9aead1022786fd91477e7e4c036d47111ca76b7586f" diff --git a/pyproject.toml b/pyproject.toml index 0bdb1f012..8f62f4943 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ alembic = "==1.13.2" amqp = "==5.2.0" beautifulsoup4 = "==4.12.3" boto3 = "^1.34.136" -botocore = "^1.34.136" +botocore = "^1.34.138" cachetools = "==5.3.3" celery = {version = "==5.4.0", extras = ["redis"]} certifi = ">=2022.12.7" From 53162de32e3e86d42a015c5dbe34921ea8143e97 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 2 Jul 2024 21:38:08 +0000 Subject: [PATCH 39/45] Bump phonenumbers from 8.13.39 to 8.13.40 Bumps [phonenumbers](https://github.com/daviddrysdale/python-phonenumbers) from 8.13.39 to 8.13.40. - [Commits](https://github.com/daviddrysdale/python-phonenumbers/compare/v8.13.39...v8.13.40) --- updated-dependencies: - dependency-name: phonenumbers dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- poetry.lock | 8 ++++---- pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index 9c5c4d5e7..51eaf7d06 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2826,13 +2826,13 @@ ptyprocess = ">=0.5" [[package]] name = "phonenumbers" -version = "8.13.39" +version = "8.13.40" description = "Python version of Google's common library for parsing, formatting, storing and validating international phone numbers." optional = false python-versions = "*" files = [ - {file = "phonenumbers-8.13.39-py2.py3-none-any.whl", hash = "sha256:3ad2d086fa71e7eef409001b9195ac54bebb0c6e3e752209b558ca192c9229a0"}, - {file = "phonenumbers-8.13.39.tar.gz", hash = "sha256:db7ca4970d206b2056231105300753b1a5b229f43416f8c2b3010e63fbb68d77"}, + {file = "phonenumbers-8.13.40-py2.py3-none-any.whl", hash = "sha256:9582752c20a1da5ec4449f7f97542bf8a793c8e2fec0ab57f767177bb8fc0b1d"}, + {file = "phonenumbers-8.13.40.tar.gz", hash = "sha256:f137c2848b8e83dd064b71881b65680584417efa202177fd330e2f7ff6c68113"}, ] [[package]] @@ -4752,4 +4752,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.12.2" -content-hash = "5ac3c87860d9d9ccce2ed9aead1022786fd91477e7e4c036d47111ca76b7586f" +content-hash = "cdc89ac0f5e246e8f15c9e6c434a4838324b8e7e0c4987ab355c93a9c4a5fa00" diff --git a/pyproject.toml b/pyproject.toml index 8f62f4943..fa2331c1e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,7 +55,7 @@ geojson = "^3.1.0" govuk-bank-holidays = "^0.14" numpy = "^1.26.4" ordered-set = "^4.1.0" -phonenumbers = "^8.13.39" +phonenumbers = "^8.13.40" python-json-logger = "^2.0.7" pytz = "^2024.1" regex = "^2024.5.15" From b436de3b23bc84d65006fc7f3e690af88c0f5fa6 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Wed, 3 Jul 2024 07:49:59 -0700 Subject: [PATCH 40/45] fix tuple --- app/commands.py | 18 ++++++++---------- poetry.lock | 8 ++------ 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/app/commands.py b/app/commands.py index d8b9d283e..c97a2f774 100644 --- a/app/commands.py +++ b/app/commands.py @@ -593,10 +593,10 @@ def process_row_from_job(job_id, job_row_number): @click.option("-f", "--csv_filename", required=True, help="csv file name") def download_csv_file_by_name(csv_filename): - bucket_name = (current_app.config["CSV_UPLOAD_BUCKET"]["bucket"],) - access_key = (current_app.config["CSV_UPLOAD_BUCKET"]["access_key_id"],) - secret = (current_app.config["CSV_UPLOAD_BUCKET"]["secret_access_key"],) - region = (current_app.config["CSV_UPLOAD_BUCKET"]["region"],) + bucket_name = current_app.config["CSV_UPLOAD_BUCKET"]["bucket"] + access_key = current_app.config["CSV_UPLOAD_BUCKET"]["access_key_id"] + secret = current_app.config["CSV_UPLOAD_BUCKET"]["secret_access_key"] + region = current_app.config["CSV_UPLOAD_BUCKET"]["region"] print(s3.get_s3_file(bucket_name, csv_filename, access_key, secret, region)) @@ -865,14 +865,12 @@ def promote_user_to_platform_admin(user_email_address): @notify_command(name="purge-csv-bucket") def purge_csv_bucket(): - bucket_name = (current_app.config["CSV_UPLOAD_BUCKET"]["bucket"],) - access_key = (current_app.config["CSV_UPLOAD_BUCKET"]["access_key_id"],) - secret = (current_app.config["CSV_UPLOAD_BUCKET"]["secret_access_key"],) - region = (current_app.config["CSV_UPLOAD_BUCKET"]["region"],) + bucket_name = current_app.config["CSV_UPLOAD_BUCKET"]["bucket"] + access_key = current_app.config["CSV_UPLOAD_BUCKET"]["access_key_id"] + secret = current_app.config["CSV_UPLOAD_BUCKET"]["secret_access_key"] + region = current_app.config["CSV_UPLOAD_BUCKET"]["region"] - print("ABOUT TO RUN PURGE CSV BUCKET") s3.purge_bucket(bucket_name, access_key, secret, region) - print("RAN PURGE CSV BUCKET") """ diff --git a/poetry.lock b/poetry.lock index 51eaf7d06..990fdec8d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. [[package]] name = "aiohttp" @@ -2098,13 +2098,9 @@ files = [ {file = "lxml-5.2.2-cp36-cp36m-win_amd64.whl", hash = "sha256:edcfa83e03370032a489430215c1e7783128808fd3e2e0a3225deee278585196"}, {file = "lxml-5.2.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:28bf95177400066596cdbcfc933312493799382879da504633d16cf60bba735b"}, {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3a745cc98d504d5bd2c19b10c79c61c7c3df9222629f1b6210c0368177589fb8"}, - {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1b590b39ef90c6b22ec0be925b211298e810b4856909c8ca60d27ffbca6c12e6"}, {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b336b0416828022bfd5a2e3083e7f5ba54b96242159f83c7e3eebaec752f1716"}, - {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_28_aarch64.whl", hash = "sha256:c2faf60c583af0d135e853c86ac2735ce178f0e338a3c7f9ae8f622fd2eb788c"}, {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:4bc6cb140a7a0ad1f7bc37e018d0ed690b7b6520ade518285dc3171f7a117905"}, - {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7ff762670cada8e05b32bf1e4dc50b140790909caa8303cfddc4d702b71ea184"}, {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:57f0a0bbc9868e10ebe874e9f129d2917750adf008fe7b9c1598c0fbbfdde6a6"}, - {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:a6d2092797b388342c1bc932077ad232f914351932353e2e8706851c870bca1f"}, {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:60499fe961b21264e17a471ec296dcbf4365fbea611bf9e303ab69db7159ce61"}, {file = "lxml-5.2.2-cp37-cp37m-win32.whl", hash = "sha256:d9b342c76003c6b9336a80efcc766748a333573abf9350f4094ee46b006ec18f"}, {file = "lxml-5.2.2-cp37-cp37m-win_amd64.whl", hash = "sha256:b16db2770517b8799c79aa80f4053cd6f8b716f21f8aca962725a9565ce3ee40"}, @@ -2493,6 +2489,7 @@ files = [ {file = "msgpack-1.0.8-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5fbb160554e319f7b22ecf530a80a3ff496d38e8e07ae763b9e82fadfe96f273"}, {file = "msgpack-1.0.8-cp39-cp39-win32.whl", hash = "sha256:f9af38a89b6a5c04b7d18c492c8ccf2aee7048aff1ce8437c4683bb5a1df893d"}, {file = "msgpack-1.0.8-cp39-cp39-win_amd64.whl", hash = "sha256:ed59dd52075f8fc91da6053b12e8c89e37aa043f8986efd89e61fae69dc1b011"}, + {file = "msgpack-1.0.8-py3-none-any.whl", hash = "sha256:24f727df1e20b9876fa6e95f840a2a2651e34c0ad147676356f4bf5fbb0206ca"}, {file = "msgpack-1.0.8.tar.gz", hash = "sha256:95c02b0e27e706e48d0e5426d1710ca78e0f0628d6e89d5b5a5b91a5f12274f3"}, ] @@ -3475,7 +3472,6 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, From 723b89da516a6a1bce33c581f13cf1bd37d0aabf Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Wed, 3 Jul 2024 08:44:13 -0700 Subject: [PATCH 41/45] fix command syntax --- docs/all.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/docs/all.md b/docs/all.md index 3d66dd5cf..a59bd761c 100644 --- a/docs/all.md +++ b/docs/all.md @@ -505,7 +505,7 @@ flask command purge_functional_test_data -u Running on cloud.gov: ``` -cf run-task notify-api "flask command purge_functional_test_data -u " +cf run-task notify-api --comand "flask command purge_functional_test_data -u " ``` @@ -1339,9 +1339,15 @@ In the api logs, search by job_id. Either you will see evidence of the job fail ## Viewing the csv file -If you need to view the questionable csv file, run the following command: +If you need to view the questionable csv file on production, run the following command: ``` -cf run-task notify-api "flask command download_csv_file_by_name -f " +cf run-task notify-api-production --command "flask command download-csv-file-by-name -f " +``` + +locally, just do: + +``` +poetry run flask command download-csv-file-by-name -f ``` From 7f33e4445cdbba7d6bf915ff3d92ce8c5a9480d7 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Wed, 3 Jul 2024 10:18:17 -0700 Subject: [PATCH 42/45] add debug steps --- docs/all.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/all.md b/docs/all.md index a59bd761c..d98f0d336 100644 --- a/docs/all.md +++ b/docs/all.md @@ -505,7 +505,7 @@ flask command purge_functional_test_data -u Running on cloud.gov: ``` -cf run-task notify-api --comand "flask command purge_functional_test_data -u " +cf run-task notify-api --command "flask command purge_functional_test_data -u " ``` @@ -1351,3 +1351,12 @@ locally, just do: ``` poetry run flask command download-csv-file-by-name -f ``` + +## Debug steps + +1. Either send a message and capture the csv file name, or get a csv file name from a user +2. Using the log tool at logs.fr.cloud.gov, use filters to limit what you're searching on (cf.app is 'notify-admin-production' for example) and then search with the csv file name in double quotes over the relevant time period (last 5 minutes if you just sent a message, or else whatever time the user sent at) +3. When you find the log line, you should also find the job_id and the s3 file location. Save these somewhere. +4. To get the csv file contents, you can run the command above. This command currently prints to the notify-api log, so after you run the command, +you need to search in notify-api-production for the last 5 minutes with the logs sorted by timestamp. The contents of the csv file unfortunately appear on separate lines so it's very important to sort by time. +5. If you want to see where the message actually failed, search with cf.app is notify-api-production using the job_id that you saved in step #3. If you get far enough, you might see one of the log lines has a message_id. If you see it, you can switch and search on that, which should tell you what happened in AWS (success or failure). From f901f6d04afa5a178a9428414ed0e070e321d738 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 3 Jul 2024 21:50:27 +0000 Subject: [PATCH 43/45] Bump boto3 from 1.34.136 to 1.34.138 Bumps [boto3](https://github.com/boto/boto3) from 1.34.136 to 1.34.138. - [Release notes](https://github.com/boto/boto3/releases) - [Commits](https://github.com/boto/boto3/compare/1.34.136...1.34.138) --- updated-dependencies: - dependency-name: boto3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- poetry.lock | 18 +++++++++++------- pyproject.toml | 2 +- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/poetry.lock b/poetry.lock index 990fdec8d..daf2521fa 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. [[package]] name = "aiohttp" @@ -403,17 +403,17 @@ files = [ [[package]] name = "boto3" -version = "1.34.136" +version = "1.34.138" description = "The AWS SDK for Python" optional = false python-versions = ">=3.8" files = [ - {file = "boto3-1.34.136-py3-none-any.whl", hash = "sha256:d41037e2c680ab8d6c61a0a4ee6bf1fdd9e857f43996672830a95d62d6f6fa79"}, - {file = "boto3-1.34.136.tar.gz", hash = "sha256:0314e6598f59ee0f34eb4e6d1a0f69fa65c146d2b88a6e837a527a9956ec2731"}, + {file = "boto3-1.34.138-py3-none-any.whl", hash = "sha256:81518aa95fad71279411fb5c94da4b4a554a5d53fc876faca62b7b5c8737f1cb"}, + {file = "boto3-1.34.138.tar.gz", hash = "sha256:f79c15e33eb7706f197d98d828b193cf0891966682ad3ec5e900f6f9e7362e35"}, ] [package.dependencies] -botocore = ">=1.34.136,<1.35.0" +botocore = ">=1.34.138,<1.35.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.10.0,<0.11.0" @@ -2098,9 +2098,13 @@ files = [ {file = "lxml-5.2.2-cp36-cp36m-win_amd64.whl", hash = "sha256:edcfa83e03370032a489430215c1e7783128808fd3e2e0a3225deee278585196"}, {file = "lxml-5.2.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:28bf95177400066596cdbcfc933312493799382879da504633d16cf60bba735b"}, {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3a745cc98d504d5bd2c19b10c79c61c7c3df9222629f1b6210c0368177589fb8"}, + {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1b590b39ef90c6b22ec0be925b211298e810b4856909c8ca60d27ffbca6c12e6"}, {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b336b0416828022bfd5a2e3083e7f5ba54b96242159f83c7e3eebaec752f1716"}, + {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_28_aarch64.whl", hash = "sha256:c2faf60c583af0d135e853c86ac2735ce178f0e338a3c7f9ae8f622fd2eb788c"}, {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:4bc6cb140a7a0ad1f7bc37e018d0ed690b7b6520ade518285dc3171f7a117905"}, + {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7ff762670cada8e05b32bf1e4dc50b140790909caa8303cfddc4d702b71ea184"}, {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:57f0a0bbc9868e10ebe874e9f129d2917750adf008fe7b9c1598c0fbbfdde6a6"}, + {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:a6d2092797b388342c1bc932077ad232f914351932353e2e8706851c870bca1f"}, {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:60499fe961b21264e17a471ec296dcbf4365fbea611bf9e303ab69db7159ce61"}, {file = "lxml-5.2.2-cp37-cp37m-win32.whl", hash = "sha256:d9b342c76003c6b9336a80efcc766748a333573abf9350f4094ee46b006ec18f"}, {file = "lxml-5.2.2-cp37-cp37m-win_amd64.whl", hash = "sha256:b16db2770517b8799c79aa80f4053cd6f8b716f21f8aca962725a9565ce3ee40"}, @@ -2489,7 +2493,6 @@ files = [ {file = "msgpack-1.0.8-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5fbb160554e319f7b22ecf530a80a3ff496d38e8e07ae763b9e82fadfe96f273"}, {file = "msgpack-1.0.8-cp39-cp39-win32.whl", hash = "sha256:f9af38a89b6a5c04b7d18c492c8ccf2aee7048aff1ce8437c4683bb5a1df893d"}, {file = "msgpack-1.0.8-cp39-cp39-win_amd64.whl", hash = "sha256:ed59dd52075f8fc91da6053b12e8c89e37aa043f8986efd89e61fae69dc1b011"}, - {file = "msgpack-1.0.8-py3-none-any.whl", hash = "sha256:24f727df1e20b9876fa6e95f840a2a2651e34c0ad147676356f4bf5fbb0206ca"}, {file = "msgpack-1.0.8.tar.gz", hash = "sha256:95c02b0e27e706e48d0e5426d1710ca78e0f0628d6e89d5b5a5b91a5f12274f3"}, ] @@ -3472,6 +3475,7 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, @@ -4748,4 +4752,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.12.2" -content-hash = "cdc89ac0f5e246e8f15c9e6c434a4838324b8e7e0c4987ab355c93a9c4a5fa00" +content-hash = "88443bf801c8f2dd1a554e94e4ace18f54d71e95c43bd3ca0163f04ef9eb0a7b" diff --git a/pyproject.toml b/pyproject.toml index fa2331c1e..1b42f95f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ python = "^3.12.2" alembic = "==1.13.2" amqp = "==5.2.0" beautifulsoup4 = "==4.12.3" -boto3 = "^1.34.136" +boto3 = "^1.34.138" botocore = "^1.34.138" cachetools = "==5.3.3" celery = {version = "==5.4.0", extras = ["redis"]} From ef4eee56454408289dff05f5ab920b16844989f5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 5 Jul 2024 12:26:23 +0000 Subject: [PATCH 44/45] Bump botocore from 1.34.138 to 1.34.139 Bumps [botocore](https://github.com/boto/botocore) from 1.34.138 to 1.34.139. - [Changelog](https://github.com/boto/botocore/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/botocore/compare/1.34.138...1.34.139) --- updated-dependencies: - dependency-name: botocore dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- poetry.lock | 18 +++++++++--------- pyproject.toml | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/poetry.lock b/poetry.lock index daf2521fa..978ba838f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. [[package]] name = "aiohttp" @@ -204,17 +204,17 @@ tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "p [[package]] name = "awscli" -version = "1.33.20" +version = "1.33.21" description = "Universal Command Line Environment for AWS." optional = false python-versions = ">=3.8" files = [ - {file = "awscli-1.33.20-py3-none-any.whl", hash = "sha256:bcb0d8e10ddeb966a2d87b7e5b564c9d849ada0a0819e1fd14e6afc9e319abf9"}, - {file = "awscli-1.33.20.tar.gz", hash = "sha256:d8167ac869b1e3750790944c56d5fd2f468d75163c3d0cbcf78444158541d9b5"}, + {file = "awscli-1.33.21-py3-none-any.whl", hash = "sha256:92e5f8a0f5e3497459f1711ef4044ac1e60ff8d2058e889c63aa6929d5c67459"}, + {file = "awscli-1.33.21.tar.gz", hash = "sha256:d0a7209e323c85b28d85cffa9470fff664d5f861bb3b5bda843329e0836f5760"}, ] [package.dependencies] -botocore = "1.34.138" +botocore = "1.34.139" colorama = ">=0.2.5,<0.4.7" docutils = ">=0.10,<0.17" PyYAML = ">=3.10,<6.1" @@ -422,13 +422,13 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.34.138" +version = "1.34.139" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">=3.8" files = [ - {file = "botocore-1.34.138-py3-none-any.whl", hash = "sha256:84e96a954c39a6f09cae4ea95b2ae582b5ae01b5040c92507b60509c9be5377a"}, - {file = "botocore-1.34.138.tar.gz", hash = "sha256:f558bbea96c4a4abbaeeedc477dabb00902311ba1ca6327974a6819b9f384920"}, + {file = "botocore-1.34.139-py3-none-any.whl", hash = "sha256:dd1e085d4caa2a4c1b7d83e3bc51416111c8238a35d498e9d3b04f3b63b086ba"}, + {file = "botocore-1.34.139.tar.gz", hash = "sha256:df023d8cf8999d574214dad4645cb90f9d2ccd1494f6ee2b57b1ab7522f6be77"}, ] [package.dependencies] @@ -4752,4 +4752,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.12.2" -content-hash = "88443bf801c8f2dd1a554e94e4ace18f54d71e95c43bd3ca0163f04ef9eb0a7b" +content-hash = "74d41976bb5028dce7b953ee4c2f6108a46215c8e48d2a4b2152d0277a63b395" diff --git a/pyproject.toml b/pyproject.toml index 1b42f95f3..627c97d67 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ alembic = "==1.13.2" amqp = "==5.2.0" beautifulsoup4 = "==4.12.3" boto3 = "^1.34.138" -botocore = "^1.34.138" +botocore = "^1.34.139" cachetools = "==5.3.3" celery = {version = "==5.4.0", extras = ["redis"]} certifi = ">=2022.12.7" From f8fb65d6d6af015e64f88a06bd02bdec66338817 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 5 Jul 2024 12:45:41 +0000 Subject: [PATCH 45/45] Bump certifi from 2024.6.2 to 2024.7.4 Bumps [certifi](https://github.com/certifi/python-certifi) from 2024.6.2 to 2024.7.4. - [Commits](https://github.com/certifi/python-certifi/compare/2024.06.02...2024.07.04) --- updated-dependencies: - dependency-name: certifi dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 978ba838f..3f4f7ba30 100644 --- a/poetry.lock +++ b/poetry.lock @@ -553,13 +553,13 @@ zstd = ["zstandard (==0.22.0)"] [[package]] name = "certifi" -version = "2024.6.2" +version = "2024.7.4" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" files = [ - {file = "certifi-2024.6.2-py3-none-any.whl", hash = "sha256:ddc6c8ce995e6987e7faf5e3f1b02b302836a0e5d98ece18392cb1a36c72ad56"}, - {file = "certifi-2024.6.2.tar.gz", hash = "sha256:3cd43f1c6fa7dedc5899d69d3ad0398fd018ad1a17fba83ddaf78aa46c747516"}, + {file = "certifi-2024.7.4-py3-none-any.whl", hash = "sha256:c198e21b1289c2ab85ee4e67bb4b4ef3ead0892059901a8d5b622f24a1101e90"}, + {file = "certifi-2024.7.4.tar.gz", hash = "sha256:5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b"}, ] [[package]]