From 2454a6e94bf1334e2d53a82ded7505587cd40f2a Mon Sep 17 00:00:00 2001 From: Andrew Shumway Date: Mon, 15 Apr 2024 10:38:27 -0600 Subject: [PATCH 01/28] poetry lock --- poetry.lock | 1 - 1 file changed, 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index 21064bf51..4e46a0a61 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1516,7 +1516,6 @@ files = [ {file = "msgpack-1.0.8-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5fbb160554e319f7b22ecf530a80a3ff496d38e8e07ae763b9e82fadfe96f273"}, {file = "msgpack-1.0.8-cp39-cp39-win32.whl", hash = "sha256:f9af38a89b6a5c04b7d18c492c8ccf2aee7048aff1ce8437c4683bb5a1df893d"}, {file = "msgpack-1.0.8-cp39-cp39-win_amd64.whl", hash = "sha256:ed59dd52075f8fc91da6053b12e8c89e37aa043f8986efd89e61fae69dc1b011"}, - {file = "msgpack-1.0.8-py3-none-any.whl", hash = "sha256:24f727df1e20b9876fa6e95f840a2a2651e34c0ad147676356f4bf5fbb0206ca"}, {file = "msgpack-1.0.8.tar.gz", hash = "sha256:95c02b0e27e706e48d0e5426d1710ca78e0f0628d6e89d5b5a5b91a5f12274f3"}, ] From 98f12980add604fb9cddb20e4a12d060c1368f3b Mon Sep 17 00:00:00 2001 From: Andrew Shumway Date: Mon, 22 Apr 2024 10:48:46 -0600 Subject: [PATCH 02/28] poetry lock --- poetry.lock | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index 0e93fd083..fc284fe31 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1211,6 +1211,7 @@ description = "Powerful and Pythonic XML processing library combining libxml2/li optional = false python-versions = ">=3.6" files = [ + {file = "lxml-5.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:704f5572ff473a5f897745abebc6df40f22d4133c1e0a1f124e4f2bd3330ff7e"}, {file = "lxml-5.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9d3c0f8567ffe7502d969c2c1b809892dc793b5d0665f602aad19895f8d508da"}, {file = "lxml-5.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5fcfbebdb0c5d8d18b84118842f31965d59ee3e66996ac842e21f957eb76138c"}, {file = "lxml-5.1.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2f37c6d7106a9d6f0708d4e164b707037b7380fcd0b04c5bd9cae1fb46a856fb"}, @@ -1220,6 +1221,7 @@ files = [ {file = "lxml-5.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:82bddf0e72cb2af3cbba7cec1d2fd11fda0de6be8f4492223d4a268713ef2147"}, {file = "lxml-5.1.0-cp310-cp310-win32.whl", hash = "sha256:b66aa6357b265670bb574f050ffceefb98549c721cf28351b748be1ef9577d93"}, {file = "lxml-5.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:4946e7f59b7b6a9e27bef34422f645e9a368cb2be11bf1ef3cafc39a1f6ba68d"}, + {file = "lxml-5.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:14deca1460b4b0f6b01f1ddc9557704e8b365f55c63070463f6c18619ebf964f"}, {file = "lxml-5.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ed8c3d2cd329bf779b7ed38db176738f3f8be637bb395ce9629fc76f78afe3d4"}, {file = "lxml-5.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:436a943c2900bb98123b06437cdd30580a61340fbdb7b28aaf345a459c19046a"}, {file = "lxml-5.1.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:acb6b2f96f60f70e7f34efe0c3ea34ca63f19ca63ce90019c6cbca6b676e81fa"}, @@ -1229,6 +1231,7 @@ files = [ {file = "lxml-5.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f4c9bda132ad108b387c33fabfea47866af87f4ea6ffb79418004f0521e63204"}, {file = "lxml-5.1.0-cp311-cp311-win32.whl", hash = "sha256:bc64d1b1dab08f679fb89c368f4c05693f58a9faf744c4d390d7ed1d8223869b"}, {file = "lxml-5.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:a5ab722ae5a873d8dcee1f5f45ddd93c34210aed44ff2dc643b5025981908cda"}, + {file = "lxml-5.1.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:9aa543980ab1fbf1720969af1d99095a548ea42e00361e727c58a40832439114"}, {file = "lxml-5.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:6f11b77ec0979f7e4dc5ae081325a2946f1fe424148d3945f943ceaede98adb8"}, {file = "lxml-5.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a36c506e5f8aeb40680491d39ed94670487ce6614b9d27cabe45d94cd5d63e1e"}, {file = "lxml-5.1.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f643ffd2669ffd4b5a3e9b41c909b72b2a1d5e4915da90a77e119b8d48ce867a"}, @@ -1254,8 +1257,8 @@ files = [ {file = "lxml-5.1.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8f52fe6859b9db71ee609b0c0a70fea5f1e71c3462ecf144ca800d3f434f0764"}, {file = "lxml-5.1.0-cp37-cp37m-win32.whl", hash = "sha256:d42e3a3fc18acc88b838efded0e6ec3edf3e328a58c68fbd36a7263a874906c8"}, {file = "lxml-5.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:eac68f96539b32fce2c9b47eb7c25bb2582bdaf1bbb360d25f564ee9e04c542b"}, + {file = "lxml-5.1.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:ae15347a88cf8af0949a9872b57a320d2605ae069bcdf047677318bc0bba45b1"}, {file = "lxml-5.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c26aab6ea9c54d3bed716b8851c8bfc40cb249b8e9880e250d1eddde9f709bf5"}, - {file = "lxml-5.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:cfbac9f6149174f76df7e08c2e28b19d74aed90cad60383ad8671d3af7d0502f"}, {file = "lxml-5.1.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:342e95bddec3a698ac24378d61996b3ee5ba9acfeb253986002ac53c9a5f6f84"}, {file = "lxml-5.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:725e171e0b99a66ec8605ac77fa12239dbe061482ac854d25720e2294652eeaa"}, {file = "lxml-5.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d184e0d5c918cff04cdde9dbdf9600e960161d773666958c9d7b565ccc60c45"}, @@ -1263,6 +1266,7 @@ files = [ {file = "lxml-5.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6d48fc57e7c1e3df57be5ae8614bab6d4e7b60f65c5457915c26892c41afc59e"}, {file = "lxml-5.1.0-cp38-cp38-win32.whl", hash = "sha256:7ec465e6549ed97e9f1e5ed51c657c9ede767bc1c11552f7f4d022c4df4a977a"}, {file = "lxml-5.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:b21b4031b53d25b0858d4e124f2f9131ffc1530431c6d1321805c90da78388d1"}, + {file = "lxml-5.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:52427a7eadc98f9e62cb1368a5079ae826f94f05755d2d567d93ee1bc3ceb354"}, {file = "lxml-5.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6a2a2c724d97c1eb8cf966b16ca2915566a4904b9aad2ed9a09c748ffe14f969"}, {file = "lxml-5.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:843b9c835580d52828d8f69ea4302537337a21e6b4f1ec711a52241ba4a824f3"}, {file = "lxml-5.1.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9b99f564659cfa704a2dd82d0684207b1aadf7d02d33e54845f9fc78e06b7581"}, From 65899f61711018b94e43355e00685b9329745354 Mon Sep 17 00:00:00 2001 From: Andrew Shumway Date: Fri, 26 Apr 2024 10:15:44 -0600 Subject: [PATCH 03/28] poetry lock --- poetry.lock | 1 - 1 file changed, 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index f70d06207..fc284fe31 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1517,7 +1517,6 @@ files = [ {file = "msgpack-1.0.8-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5fbb160554e319f7b22ecf530a80a3ff496d38e8e07ae763b9e82fadfe96f273"}, {file = "msgpack-1.0.8-cp39-cp39-win32.whl", hash = "sha256:f9af38a89b6a5c04b7d18c492c8ccf2aee7048aff1ce8437c4683bb5a1df893d"}, {file = "msgpack-1.0.8-cp39-cp39-win_amd64.whl", hash = "sha256:ed59dd52075f8fc91da6053b12e8c89e37aa043f8986efd89e61fae69dc1b011"}, - {file = "msgpack-1.0.8-py3-none-any.whl", hash = "sha256:24f727df1e20b9876fa6e95f840a2a2651e34c0ad147676356f4bf5fbb0206ca"}, {file = "msgpack-1.0.8.tar.gz", hash = "sha256:95c02b0e27e706e48d0e5426d1710ca78e0f0628d6e89d5b5a5b91a5f12274f3"}, ] From 5506da56e219753d3a4d4c61d279f0f4b19092f0 Mon Sep 17 00:00:00 2001 From: Andrew Shumway Date: Wed, 8 May 2024 09:05:27 -0600 Subject: [PATCH 04/28] poetry lock --- poetry.lock | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index bb92d1c04..1b201f09b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. [[package]] name = "ago" @@ -1278,6 +1278,7 @@ files = [ {file = "lxml-5.2.1-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c38d7b9a690b090de999835f0443d8aa93ce5f2064035dfc48f27f02b4afc3d0"}, {file = "lxml-5.2.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5670fb70a828663cc37552a2a85bf2ac38475572b0e9b91283dc09efb52c41d1"}, {file = "lxml-5.2.1-cp36-cp36m-manylinux_2_28_x86_64.whl", hash = "sha256:958244ad566c3ffc385f47dddde4145088a0ab893504b54b52c041987a8c1863"}, + {file = "lxml-5.2.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:b6241d4eee5f89453307c2f2bfa03b50362052ca0af1efecf9fef9a41a22bb4f"}, {file = "lxml-5.2.1-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:2a66bf12fbd4666dd023b6f51223aed3d9f3b40fef06ce404cb75bafd3d89536"}, {file = "lxml-5.2.1-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:9123716666e25b7b71c4e1789ec829ed18663152008b58544d95b008ed9e21e9"}, {file = "lxml-5.2.1-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:0c3f67e2aeda739d1cc0b1102c9a9129f7dc83901226cc24dd72ba275ced4218"}, From 8e6b3fa39e336389eb1163a3710e251b9c3c3758 Mon Sep 17 00:00:00 2001 From: Andrew Shumway Date: Wed, 29 May 2024 10:15:11 -0600 Subject: [PATCH 05/28] Poetry lock --- poetry.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/poetry.lock b/poetry.lock index 90f743b80..96696c8dc 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1297,6 +1297,7 @@ files = [ {file = "lxml-5.2.1-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c38d7b9a690b090de999835f0443d8aa93ce5f2064035dfc48f27f02b4afc3d0"}, {file = "lxml-5.2.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5670fb70a828663cc37552a2a85bf2ac38475572b0e9b91283dc09efb52c41d1"}, {file = "lxml-5.2.1-cp36-cp36m-manylinux_2_28_x86_64.whl", hash = "sha256:958244ad566c3ffc385f47dddde4145088a0ab893504b54b52c041987a8c1863"}, + {file = "lxml-5.2.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:b6241d4eee5f89453307c2f2bfa03b50362052ca0af1efecf9fef9a41a22bb4f"}, {file = "lxml-5.2.1-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:2a66bf12fbd4666dd023b6f51223aed3d9f3b40fef06ce404cb75bafd3d89536"}, {file = "lxml-5.2.1-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:9123716666e25b7b71c4e1789ec829ed18663152008b58544d95b008ed9e21e9"}, {file = "lxml-5.2.1-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:0c3f67e2aeda739d1cc0b1102c9a9129f7dc83901226cc24dd72ba275ced4218"}, From a401562735987a065f5258fea5a424a27cc746c5 Mon Sep 17 00:00:00 2001 From: Andrew Shumway Date: Mon, 10 Jun 2024 10:15:22 -0600 Subject: [PATCH 06/28] Add report options 1/3/5-day --- .ds.baseline | 4 ++-- app/config.py | 7 +++++- app/main/views/jobs.py | 32 +++++++++++++++++++++----- app/main/views/notifications.py | 4 ++-- app/models/service.py | 7 ++++-- app/templates/views/notifications.html | 14 ++++++++++- tests/app/main/views/test_activity.py | 14 +++++++---- 7 files changed, 63 insertions(+), 19 deletions(-) diff --git a/.ds.baseline b/.ds.baseline index cec28396c..f95942edb 100644 --- a/.ds.baseline +++ b/.ds.baseline @@ -169,7 +169,7 @@ "filename": "app/config.py", "hashed_secret": "577a4c667e4af8682ca431857214b3a920883efc", "is_verified": false, - "line_number": 111, + "line_number": 116, "is_secret": false } ], @@ -710,5 +710,5 @@ } ] }, - "generated_at": "2024-05-20T16:03:05Z" + "generated_at": "2024-06-10T16:09:52Z" } diff --git a/app/config.py b/app/config.py index 8ec4db186..2a0f0c3e7 100644 --- a/app/config.py +++ b/app/config.py @@ -53,7 +53,12 @@ class Config(object): PERMANENT_SESSION_LIFETIME = 1800 # 30 Minutes SEND_FILE_MAX_AGE_DEFAULT = 365 * 24 * 60 * 60 # 1 year REPLY_TO_EMAIL_ADDRESS_VALIDATION_TIMEOUT = 45 - ACTIVITY_STATS_LIMIT_DAYS = 7 + ACTIVITY_STATS_LIMIT_DAYS = { + "one_day": 1, + "three_day": 3, + "five_day": 5, + "seven_day": 7, + } SESSION_COOKIE_HTTPONLY = True SESSION_COOKIE_NAME = "notify_admin_session" SESSION_COOKIE_SECURE = True diff --git a/app/main/views/jobs.py b/app/main/views/jobs.py index 010d19b26..6608ac419 100644 --- a/app/main/views/jobs.py +++ b/app/main/views/jobs.py @@ -143,11 +143,33 @@ def view_notifications(service_id, message_type=None): True: ["reference"], False: [], }.get(bool(current_service.api_keys)), - download_link=url_for( + download_link_one_day=url_for( ".download_notifications_csv", service_id=current_service.id, message_type=message_type, status=request.args.get("status"), + number_of_days="one_day", + ), + download_link_three_day=url_for( + ".download_notifications_csv", + service_id=current_service.id, + message_type=message_type, + status=request.args.get("status"), + number_of_days="three_day", + ), + download_link_five_day=url_for( + ".download_notifications_csv", + service_id=current_service.id, + message_type=message_type, + status=request.args.get("status"), + number_of_days="five_day", + ), + download_link_seven_day=url_for( + ".download_notifications_csv", + service_id=current_service.id, + message_type=message_type, + status=request.args.get("status"), + number_of_days="seven_day", ), ) @@ -183,10 +205,9 @@ def get_notifications(service_id, message_type, status_override=None): # noqa filter_args["status"] = set_status_filters(filter_args) service_data_retention_days = None search_term = request.form.get("to", "") - if message_type is not None: service_data_retention_days = current_service.get_days_of_retention( - message_type + message_type, number_of_days="seven_day" ) if request.path.endswith("csv") and current_user.has_permissions("view_activity"): @@ -212,7 +233,6 @@ def get_notifications(service_id, message_type, status_override=None): # noqa ) url_args = {"message_type": message_type, "status": request.args.get("status")} prev_page = None - if "links" in notifications and notifications["links"].get("prev", None): prev_page = generate_previous_dict( "main.view_notifications", service_id, page, url_args=url_args @@ -233,7 +253,6 @@ def get_notifications(service_id, message_type, status_override=None): # noqa ) else: download_link = None - return { "service_data_retention_days": service_data_retention_days, "counts": render_template( @@ -362,6 +381,7 @@ def get_job_partials(job): filter_args = parse_filter_args(request.args) filter_args["status"] = set_status_filters(filter_args) notifications = job.get_notifications(status=filter_args["status"]) + number_of_days = "seven_day" counts = render_template( "partials/count.html", counts=_get_job_counts(job), @@ -371,7 +391,7 @@ def get_job_partials(job): ), ) service_data_retention_days = current_service.get_days_of_retention( - job.template_type + job.template_type, number_of_days ) if request.referrer is not None: diff --git a/app/main/views/notifications.py b/app/main/views/notifications.py index ac05e05ff..e41708b8c 100644 --- a/app/main/views/notifications.py +++ b/app/main/views/notifications.py @@ -137,9 +137,9 @@ def get_all_personalisation_from_notification(notification): def download_notifications_csv(service_id): filter_args = parse_filter_args(request.args) filter_args["status"] = set_status_filters(filter_args) - + number_of_days = request.args["number_of_days"] service_data_retention_days = current_service.get_days_of_retention( - filter_args.get("message_type")[0] + filter_args.get("message_type")[0], number_of_days ) file_time = datetime.now().strftime("%Y-%m-%d %I:%M:%S %p") file_time = f"{file_time} {get_user_preferred_timezone()}" diff --git a/app/models/service.py b/app/models/service.py index 375047d8d..e9bcf8a7d 100644 --- a/app/models/service.py +++ b/app/models/service.py @@ -390,7 +390,7 @@ class Service(JSONModel, SortByNameMixin): def get_data_retention_item(self, id): return next((dr for dr in self.data_retention if dr["id"] == id), None) - def get_days_of_retention(self, notification_type): + def get_days_of_retention(self, notification_type, number_of_days): return next( ( dr @@ -398,7 +398,10 @@ class Service(JSONModel, SortByNameMixin): if dr["notification_type"] == notification_type ), {}, - ).get("days_of_retention", current_app.config["ACTIVITY_STATS_LIMIT_DAYS"]) + ).get( + "days_of_retention", + current_app.config["ACTIVITY_STATS_LIMIT_DAYS"].get(number_of_days), + ) @cached_property def organization(self): diff --git a/app/templates/views/notifications.html b/app/templates/views/notifications.html index 9f5accd43..4858382b2 100644 --- a/app/templates/views/notifications.html +++ b/app/templates/views/notifications.html @@ -64,10 +64,22 @@ {% if current_user.has_permissions('view_activity') %}

- Download this report (CSV) + Download all data last 7 days (CSV)   Data available for {{ partials.service_data_retention_days }} days

+

+ Download all data last 5 days (CSV) +   +

+

+ Download all data last 3 days (CSV) +   +

+

+ Download all data today (CSV) +   +

{% endif %} {{ ajax_block( diff --git a/tests/app/main/views/test_activity.py b/tests/app/main/views/test_activity.py index 3493aaae4..3bbe40282 100644 --- a/tests/app/main/views/test_activity.py +++ b/tests/app/main/views/test_activity.py @@ -228,12 +228,18 @@ def test_can_show_notifications_if_data_retention_not_available( url_for, ".download_notifications_csv", message_type=None, + number_of_days="seven_day", ), ), ( create_active_user_with_permissions(), {"status": "failed"}, - partial(url_for, ".download_notifications_csv", status="failed"), + partial( + url_for, + ".download_notifications_csv", + status="failed", + number_of_days="seven_day", + ), ), ( create_active_user_with_permissions(), @@ -242,15 +248,13 @@ def test_can_show_notifications_if_data_retention_not_available( url_for, ".download_notifications_csv", message_type="sms", + number_of_days="seven_day", ), ), ( create_active_user_view_permissions(), {}, - partial( - url_for, - ".download_notifications_csv", - ), + partial(url_for, ".download_notifications_csv", number_of_days="seven_day"), ), ( create_active_caseworking_user(), From 096658ab75540b40f8eaf7d4fc0306b19325c6e3 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Tue, 11 Jun 2024 13:24:05 -0700 Subject: [PATCH 07/28] Need magic PII-free debugging method for Admin --- app/main/views/send.py | 9 ++----- app/utils/csv.py | 5 ++-- notifications_utils/logging.py | 33 ++++++++++++++++------- tests/app/main/views/test_dashboard.py | 28 +++++++++---------- tests/notifications_utils/test_logging.py | 19 ++++++++----- 5 files changed, 51 insertions(+), 43 deletions(-) diff --git a/app/main/views/send.py b/app/main/views/send.py index 8bb6ce24c..1dcb8fa59 100644 --- a/app/main/views/send.py +++ b/app/main/views/send.py @@ -51,7 +51,6 @@ from app.utils.templates import get_template from app.utils.user import user_has_permissions from notifications_utils import SMS_CHAR_COUNT_LIMIT from notifications_utils.insensitive_dict import InsensitiveDict -from notifications_utils.logging import scrub from notifications_utils.recipients import RecipientCSV, first_column_headings from notifications_utils.sanitise_text import SanitiseASCII @@ -953,9 +952,7 @@ def send_notification(service_id, template_id): ) ) - current_app.logger.info( - hilite(scrub(f"Recipient for the one-off will be {recipient}")) - ) + current_app.logger.info(hilite(f"Recipient for the one-off will be {recipient}")) keys = [] values = [] for k, v in session["placeholders"].items(): @@ -996,9 +993,7 @@ def send_notification(service_id, template_id): # about report generation. current_app.logger.info( hilite( - scrub( - f"Created job to send one-off, recipient is {recipient}, job_id is {upload_id}" - ) + f"Created job to send one-off, recipient is {recipient}, job_id is {upload_id}" ) ) diff --git a/app/utils/csv.py b/app/utils/csv.py index 5c5b794de..159234786 100644 --- a/app/utils/csv.py +++ b/app/utils/csv.py @@ -7,7 +7,6 @@ from flask_login import current_user from app.models.spreadsheet import Spreadsheet from app.utils import hilite from app.utils.templates import get_sample_template -from notifications_utils.logging import scrub from notifications_utils.recipients import RecipientCSV @@ -79,7 +78,7 @@ def generate_notifications_csv(**kwargs): # hence the try/except. try: current_app.logger.info( - hilite(f"Setting up report with kwargs {scrub(json.dumps(kwargs))}") + hilite(f"Setting up report with kwargs {json.dumps(kwargs)}") ) except TypeError: pass @@ -89,7 +88,7 @@ def generate_notifications_csv(**kwargs): # we display to 999 characters, because we don't want to show the contents for reports with thousands of rows. current_app.logger.info( hilite( - f"Original csv for job_id {kwargs['job_id']}: {scrub(original_file_contents[0:999])}" + f"Original csv for job_id {kwargs['job_id']}: {original_file_contents[0:999]}" ) ) original_upload = RecipientCSV( diff --git a/notifications_utils/logging.py b/notifications_utils/logging.py index 7c56a00ad..285aeee58 100644 --- a/notifications_utils/logging.py +++ b/notifications_utils/logging.py @@ -70,6 +70,7 @@ def configure_handler(handler, app, formatter): handler.addFilter(AppNameFilter(app.config["NOTIFY_APP_NAME"])) handler.addFilter(RequestIdFilter()) handler.addFilter(ServiceIdFilter()) + handler.addFilter(PIIFilter()) return handler @@ -134,13 +135,25 @@ class JSONFormatter(BaseJSONFormatter): return log_record -def scrub(msg): - # Eventually we want to scrub all messages in all logs for phone numbers - # and email addresses, masking them. Ultimately this will probably get - # refactored into a 'SafeLogger' subclass or something, but let's start here - # with phones. - phones = re.findall("(?:\\+ *)?\\d[\\d\\- ]{7,}\\d", msg) - phones = [phone.replace("-", "").replace(" ", "") for phone in phones] - for phone in phones: - msg = msg.replace(phone, f"1XXXXX{phone[-5:]}") - return msg +class PIIFilter(logging.Filter): + def scrub(self, msg): + # Eventually we want to scrub all messages in all logs for phone numbers + # and email addresses, masking them. Ultimately this will probably get + # refactored into a 'SafeLogger' subclass or something, but let's start here + # with phones. + phones = re.findall("(?:\\+ *)?\\d[\\d\\- ]{7,}\\d", msg) + phones = [phone.replace("-", "").replace(" ", "") for phone in phones] + for phone in phones: + msg = msg.replace(phone, f"1XXXXX{phone[-5:]}") + + emails = re.findall( + r"[\w\.-]+@[\w\.-]+", msg + ) # ['alice@google.com', 'bob@abc.com'] + for email in emails: + # do something with each found email string + msg = msg.replace(email, f"XXXXX{email[-10:]}") + return msg + + def filter(self, record): + record.msg = self.scrub(record.msg) + return record diff --git a/tests/app/main/views/test_dashboard.py b/tests/app/main/views/test_dashboard.py index 285444b1b..e83dacbae 100644 --- a/tests/app/main/views/test_dashboard.py +++ b/tests/app/main/views/test_dashboard.py @@ -1893,26 +1893,22 @@ def app_with_socketio(): ( SERVICE_ONE_ID, {"start_date": "2024-01-01", "days": 7}, - {"service_id": SERVICE_ONE_ID, "start_date": "2024-01-01", "days": 7} + {"service_id": SERVICE_ONE_ID, "start_date": "2024-01-01", "days": 7}, ), ( SERVICE_TWO_ID, {"start_date": "2023-06-01", "days": 7}, - {"service_id": SERVICE_TWO_ID, "start_date": "2023-06-01", "days": 7} + {"service_id": SERVICE_TWO_ID, "start_date": "2023-06-01", "days": 7}, ), - ] + ], ) def test_fetch_daily_stats( - app_with_socketio, mocker, - service_id, - date_range, - expected_call_args + app_with_socketio, mocker, service_id, date_range, expected_call_args ): app, socketio = app_with_socketio mocker.patch( - "app.main.views.dashboard.get_stats_date_range", - return_value=date_range + "app.main.views.dashboard.get_stats_date_range", return_value=date_range ) mock_service_api = mocker.patch( @@ -1920,9 +1916,9 @@ def test_fetch_daily_stats( return_value={ date_range["start_date"]: { "email": {"delivered": 0, "failure": 0, "requested": 0}, - "sms": {"delivered": 0, "failure": 1, "requested": 1} + "sms": {"delivered": 0, "failure": 1, "requested": 1}, }, - } + }, ) client = SocketIOTestClient(app, socketio) @@ -1930,22 +1926,22 @@ def test_fetch_daily_stats( connected = client.is_connected() assert connected, "Client should be connected" - client.emit('fetch_daily_stats', service_id) + client.emit("fetch_daily_stats", service_id) received = client.get_received() assert received, "Should receive a response message" - assert received[0]['name'] == 'daily_stats_update' - assert received[0]['args'][0] == { + assert received[0]["name"] == "daily_stats_update" + assert received[0]["args"][0] == { date_range["start_date"]: { "email": {"delivered": 0, "failure": 0, "requested": 0}, - "sms": {"delivered": 0, "failure": 1, "requested": 1} + "sms": {"delivered": 0, "failure": 1, "requested": 1}, }, } mock_service_api.assert_called_once_with( service_id, start_date=expected_call_args["start_date"], - days=expected_call_args["days"] + days=expected_call_args["days"], ) finally: client.disconnect() diff --git a/tests/notifications_utils/test_logging.py b/tests/notifications_utils/test_logging.py index 858b9352b..89b220324 100644 --- a/tests/notifications_utils/test_logging.py +++ b/tests/notifications_utils/test_logging.py @@ -51,11 +51,16 @@ def test_base_json_formatter_contains_service_id(): assert service_id_filter.filter(record).service_id == "no-service-id" -def test_scrub(): - result = logging.scrub( - "This is a message with 17775554324, and also 18884449323 and also 17775554324" - ) - assert ( - result - == "This is a message with 1XXXXX54324, and also 1XXXXX49323 and also 1XXXXX54324" +def test_pii_filter(): + record = builtin_logging.LogRecord( + name="log thing", + level="info", + pathname="path", + lineno=123, + msg="phone1: 1555555555, phone2: 1555555554, email1: fake@fake.gov, email2: fake@fake2.fake.gov", + exc_info=None, + args=None, ) + pii_filter = logging.PIIFilter() + clean_msg = "phone1: 1XXXXX55555, phone2: 1XXXXX55554, email1: XXXXXe@fake.gov, email2: XXXXX2.fake.gov" + assert pii_filter.filter(record).msg == clean_msg From f6f99b612460d0bdc70ced1d8acb2a54d79e8112 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Wed, 12 Jun 2024 10:20:06 -0700 Subject: [PATCH 08/28] fix bug where only exception is passed to logger --- notifications_utils/logging.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/notifications_utils/logging.py b/notifications_utils/logging.py index 285aeee58..71c82aacb 100644 --- a/notifications_utils/logging.py +++ b/notifications_utils/logging.py @@ -141,6 +141,10 @@ class PIIFilter(logging.Filter): # and email addresses, masking them. Ultimately this will probably get # refactored into a 'SafeLogger' subclass or something, but let's start here # with phones. + + # Sometimes just an exception object is passed in for the message, skip those. + if not isinstance(msg, str): + return msg phones = re.findall("(?:\\+ *)?\\d[\\d\\- ]{7,}\\d", msg) phones = [phone.replace("-", "").replace(" ", "") for phone in phones] for phone in phones: From 4e8e1acb5b17bd9b6da9506623945c0b0e61a2a1 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 13 Jun 2024 10:17:47 -0700 Subject: [PATCH 09/28] fix email masking algorithm --- notifications_utils/logging.py | 4 +++- tests/notifications_utils/test_logging.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/notifications_utils/logging.py b/notifications_utils/logging.py index 71c82aacb..ba5c862fe 100644 --- a/notifications_utils/logging.py +++ b/notifications_utils/logging.py @@ -155,7 +155,9 @@ class PIIFilter(logging.Filter): ) # ['alice@google.com', 'bob@abc.com'] for email in emails: # do something with each found email string - msg = msg.replace(email, f"XXXXX{email[-10:]}") + email_parts = email.split("@") + masked_email = f"{email_parts[0][0:3]}XXX@{email_parts[1][0:7]}XXX" + msg = msg.replace(email, masked_email) return msg def filter(self, record): diff --git a/tests/notifications_utils/test_logging.py b/tests/notifications_utils/test_logging.py index 89b220324..1b9f999c3 100644 --- a/tests/notifications_utils/test_logging.py +++ b/tests/notifications_utils/test_logging.py @@ -62,5 +62,5 @@ def test_pii_filter(): args=None, ) pii_filter = logging.PIIFilter() - clean_msg = "phone1: 1XXXXX55555, phone2: 1XXXXX55554, email1: XXXXXe@fake.gov, email2: XXXXX2.fake.gov" + clean_msg = "phone1: 1XXXXX55555, phone2: 1XXXXX55554, email1: fakXXX@fake.goXXX, email2: fakXXX@fake2.fXXX" assert pii_filter.filter(record).msg == clean_msg From 7e533731ed8c223e3e165c684a265971d4fdd0f5 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Mon, 17 Jun 2024 11:40:13 -0700 Subject: [PATCH 10/28] fix flake 8 --- app/main/views/send.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/app/main/views/send.py b/app/main/views/send.py index 3d8bcd512..bb63f10a9 100644 --- a/app/main/views/send.py +++ b/app/main/views/send.py @@ -991,7 +991,6 @@ def send_notification(service_id, template_id): valid="True", ) - session.pop("recipient") session.pop("placeholders") @@ -1023,8 +1022,12 @@ def send_notification(service_id, template_id): job_id=upload_id, ) ) - total = notifications['total'] - current_app.logger.info(hilite(f"job_id: {upload_id} has notifications: {total} and attempts: {attempts}")) + total = notifications["total"] + current_app.logger.info( + hilite( + f"job_id: {upload_id} has notifications: {total} and attempts: {attempts}" + ) + ) return redirect( url_for( ".view_job", From 81dd764a54d4c493dda18c37f6ddcd4e57d4d213 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Fri, 21 Jun 2024 09:22:20 -0700 Subject: [PATCH 11/28] fix gunicorn config --- gunicorn_config.py | 25 ++++--------------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/gunicorn_config.py b/gunicorn_config.py index 4f928a467..f9eb4f17d 100644 --- a/gunicorn_config.py +++ b/gunicorn_config.py @@ -5,7 +5,10 @@ import multiprocessing import gunicorn # Let gunicorn figure out the right number of workers -workers = multiprocessing.cpu_count() * 2 + 1 +# The recommended formula is cpu_count() * 2 + 1 +# but we have an unusual configuration with a lot of cpus and not much memory +# so adjust it. +workers = multiprocessing.cpu_count() worker_class = "eventlet" bind = "0.0.0.0:{}".format(os.getenv("PORT")) disable_redirect_access_to_syslog = True @@ -16,23 +19,3 @@ def worker_abort(worker): worker.log.info("worker received ABORT") for stack in sys._current_frames().values(): worker.log.error("".join(traceback.format_stack(stack))) - - -# This issue is fixed in the 22.0.0 release, which we are using -# See github issue for details -# def fix_ssl_monkeypatching(): -# """ -# eventlet works by monkey-patching core IO libraries (such as ssl) to be non-blocking. However, there's currently -# a bug: In the normal socket library it may throw a timeout error as a `socket.timeout` exception. However -# eventlet.green.ssl's patch raises an ssl.SSLError('timed out',) instead. redispy handles socket.timeout but not -# ssl.SSLError, so we solve this by monkey patching the monkey patching code to raise the correct exception type -# :scream: -# https://github.com/eventlet/eventlet/issues/692 -# """ -# # this has probably already been called somewhere in gunicorn internals, however, to be sure, we invoke it again. -# # eventlet.monkey_patch can be called multiple times without issue -# eventlet.monkey_patch() -# eventlet.green.ssl.timeout_exc = socket.timeout - - -# fix_ssl_monkeypatching() From a887f3f5273ab97c7fe893a708a6bb0824e5d181 Mon Sep 17 00:00:00 2001 From: Carlo Costino Date: Mon, 24 Jun 2024 12:07:28 -0400 Subject: [PATCH 12/28] Increased production memory to 2 GB This changeset increases our memory for the API app and its corresponding workers to 2 GB each. Signed-off-by: Carlo Costino --- deploy-config/production.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy-config/production.yml b/deploy-config/production.yml index 18b5cfdcc..b38094f28 100644 --- a/deploy-config/production.yml +++ b/deploy-config/production.yml @@ -1,6 +1,6 @@ env: production instances: 2 -memory: 1G +memory: 2G public_admin_route: beta.notify.gov cloud_dot_gov_route: notify.app.cloud.gov redis_enabled: 1 From a9e4df059154c914d0fd83033d092337a1f23b6b Mon Sep 17 00:00:00 2001 From: Andrew Shumway Date: Mon, 24 Jun 2024 10:44:19 -0600 Subject: [PATCH 13/28] Todays data is only todays --- .ds.baseline | 4 ++-- app/config.py | 1 + app/main/views/jobs.py | 7 +++++++ app/templates/views/notifications.html | 6 +++++- 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/.ds.baseline b/.ds.baseline index 7b2ab5f89..73435047a 100644 --- a/.ds.baseline +++ b/.ds.baseline @@ -169,7 +169,7 @@ "filename": "app/config.py", "hashed_secret": "577a4c667e4af8682ca431857214b3a920883efc", "is_verified": false, - "line_number": 116, + "line_number": 117, "is_secret": false } ], @@ -710,5 +710,5 @@ } ] }, - "generated_at": "2024-06-05T22:01:56Z" + "generated_at": "2024-06-24T16:44:00Z" } diff --git a/app/config.py b/app/config.py index e3d7c3af9..960d6331b 100644 --- a/app/config.py +++ b/app/config.py @@ -54,6 +54,7 @@ class Config(object): SEND_FILE_MAX_AGE_DEFAULT = 365 * 24 * 60 * 60 # 1 year REPLY_TO_EMAIL_ADDRESS_VALIDATION_TIMEOUT = 45 ACTIVITY_STATS_LIMIT_DAYS = { + "today": 0, "one_day": 1, "three_day": 3, "five_day": 5, diff --git a/app/main/views/jobs.py b/app/main/views/jobs.py index 6608ac419..42a4de090 100644 --- a/app/main/views/jobs.py +++ b/app/main/views/jobs.py @@ -150,6 +150,13 @@ def view_notifications(service_id, message_type=None): status=request.args.get("status"), number_of_days="one_day", ), + download_link_today=url_for( + ".download_notifications_csv", + service_id=current_service.id, + message_type=message_type, + status=request.args.get("status"), + number_of_days="today", + ), download_link_three_day=url_for( ".download_notifications_csv", service_id=current_service.id, diff --git a/app/templates/views/notifications.html b/app/templates/views/notifications.html index 4858382b2..487e10b98 100644 --- a/app/templates/views/notifications.html +++ b/app/templates/views/notifications.html @@ -76,9 +76,13 @@ Download all data last 3 days (CSV)  

-

+ +

+ Download all data today (CSV) +  

{% endif %} From 9685a4bd2446d6390c00893a00e3ee0d7c7ed8a0 Mon Sep 17 00:00:00 2001 From: Andrew Shumway Date: Tue, 25 Jun 2024 09:04:59 -0600 Subject: [PATCH 14/28] Remove code that shouldn't be in --- app/main/views/send.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/main/views/send.py b/app/main/views/send.py index 6d0f4f928..8bb6ce24c 100644 --- a/app/main/views/send.py +++ b/app/main/views/send.py @@ -1039,8 +1039,6 @@ def send_notification(service_id, template_id): ".view_job", service_id=service_id, job_id=upload_id, - from_job=upload_id, - notification_id=notifications["notifications"][0]["id"], # used to show the final step of the tour (help=3) or not show # a back link on a just sent one off notification (help=0) help=request.args.get("help"), From 624e161292218f73c09a681887e70a0c9f3c0544 Mon Sep 17 00:00:00 2001 From: John Skiles Skinner Date: Tue, 25 Jun 2024 14:18:14 -0700 Subject: [PATCH 15/28] Upgrade cloudfoundry to v0.53.1 to match api app --- terraform/bootstrap/providers.tf | 2 +- terraform/demo/providers.tf | 2 +- terraform/development/providers.tf | 2 +- terraform/production/providers.tf | 2 +- terraform/sandbox/providers.tf | 2 +- terraform/shared/container_networking/providers.tf | 2 +- terraform/staging/providers.tf | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/terraform/bootstrap/providers.tf b/terraform/bootstrap/providers.tf index b6f27acf8..e62de0c5e 100644 --- a/terraform/bootstrap/providers.tf +++ b/terraform/bootstrap/providers.tf @@ -3,7 +3,7 @@ terraform { required_providers { cloudfoundry = { source = "cloudfoundry-community/cloudfoundry" - version = "0.53.0" + version = "0.53.1" } } } diff --git a/terraform/demo/providers.tf b/terraform/demo/providers.tf index 2ced7915f..f69e5384b 100644 --- a/terraform/demo/providers.tf +++ b/terraform/demo/providers.tf @@ -3,7 +3,7 @@ terraform { required_providers { cloudfoundry = { source = "cloudfoundry-community/cloudfoundry" - version = "0.53.0" + version = "0.53.1" } } diff --git a/terraform/development/providers.tf b/terraform/development/providers.tf index 5dcaece3e..3c699e728 100644 --- a/terraform/development/providers.tf +++ b/terraform/development/providers.tf @@ -3,7 +3,7 @@ terraform { required_providers { cloudfoundry = { source = "cloudfoundry-community/cloudfoundry" - version = "0.53.0" + version = "0.53.1" } } } diff --git a/terraform/production/providers.tf b/terraform/production/providers.tf index ed822db44..99069fcc8 100644 --- a/terraform/production/providers.tf +++ b/terraform/production/providers.tf @@ -3,7 +3,7 @@ terraform { required_providers { cloudfoundry = { source = "cloudfoundry-community/cloudfoundry" - version = "0.53.0" + version = "0.53.1" } } diff --git a/terraform/sandbox/providers.tf b/terraform/sandbox/providers.tf index 09911edc4..87c362182 100644 --- a/terraform/sandbox/providers.tf +++ b/terraform/sandbox/providers.tf @@ -3,7 +3,7 @@ terraform { required_providers { cloudfoundry = { source = "cloudfoundry-community/cloudfoundry" - version = "0.53.0" + version = "0.53.1" } } diff --git a/terraform/shared/container_networking/providers.tf b/terraform/shared/container_networking/providers.tf index 21ac567a2..01ab1f803 100644 --- a/terraform/shared/container_networking/providers.tf +++ b/terraform/shared/container_networking/providers.tf @@ -3,7 +3,7 @@ terraform { required_providers { cloudfoundry = { source = "cloudfoundry-community/cloudfoundry" - version = "0.53.0" + version = "0.53.1" } } } diff --git a/terraform/staging/providers.tf b/terraform/staging/providers.tf index d6928e61e..adbfaec2b 100644 --- a/terraform/staging/providers.tf +++ b/terraform/staging/providers.tf @@ -3,7 +3,7 @@ terraform { required_providers { cloudfoundry = { source = "cloudfoundry-community/cloudfoundry" - version = "0.53.0" + version = "0.53.1" } } From 609547116618fae636f1b5a68aa9596798493cdc Mon Sep 17 00:00:00 2001 From: John Skiles Skinner Date: Tue, 25 Jun 2024 14:24:31 -0700 Subject: [PATCH 16/28] Terraform minimum version upgrade --- terraform/bootstrap/providers.tf | 2 +- terraform/demo/providers.tf | 2 +- terraform/development/providers.tf | 2 +- terraform/production/providers.tf | 2 +- terraform/sandbox/providers.tf | 2 +- terraform/staging/providers.tf | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/terraform/bootstrap/providers.tf b/terraform/bootstrap/providers.tf index e62de0c5e..cce97ee8f 100644 --- a/terraform/bootstrap/providers.tf +++ b/terraform/bootstrap/providers.tf @@ -1,5 +1,5 @@ terraform { - required_version = "~> 1.0" + required_version = "~> 1.7" required_providers { cloudfoundry = { source = "cloudfoundry-community/cloudfoundry" diff --git a/terraform/demo/providers.tf b/terraform/demo/providers.tf index f69e5384b..2381dcd28 100644 --- a/terraform/demo/providers.tf +++ b/terraform/demo/providers.tf @@ -1,5 +1,5 @@ terraform { - required_version = "~> 1.0" + required_version = "~> 1.7" required_providers { cloudfoundry = { source = "cloudfoundry-community/cloudfoundry" diff --git a/terraform/development/providers.tf b/terraform/development/providers.tf index 3c699e728..7b9ce5c7f 100644 --- a/terraform/development/providers.tf +++ b/terraform/development/providers.tf @@ -1,5 +1,5 @@ terraform { - required_version = "~> 1.0" + required_version = "~> 1.7" required_providers { cloudfoundry = { source = "cloudfoundry-community/cloudfoundry" diff --git a/terraform/production/providers.tf b/terraform/production/providers.tf index 99069fcc8..97f543a23 100644 --- a/terraform/production/providers.tf +++ b/terraform/production/providers.tf @@ -1,5 +1,5 @@ terraform { - required_version = "~> 1.0" + required_version = "~> 1.7" required_providers { cloudfoundry = { source = "cloudfoundry-community/cloudfoundry" diff --git a/terraform/sandbox/providers.tf b/terraform/sandbox/providers.tf index 87c362182..978b10f45 100644 --- a/terraform/sandbox/providers.tf +++ b/terraform/sandbox/providers.tf @@ -1,5 +1,5 @@ terraform { - required_version = "~> 1.0" + required_version = "~> 1.7" required_providers { cloudfoundry = { source = "cloudfoundry-community/cloudfoundry" diff --git a/terraform/staging/providers.tf b/terraform/staging/providers.tf index adbfaec2b..05d8b90d3 100644 --- a/terraform/staging/providers.tf +++ b/terraform/staging/providers.tf @@ -1,5 +1,5 @@ terraform { - required_version = "~> 1.0" + required_version = "~> 1.7" required_providers { cloudfoundry = { source = "cloudfoundry-community/cloudfoundry" From 382c1910c91b1eadc613eba3eee5d7ff5390d936 Mon Sep 17 00:00:00 2001 From: John Skiles Skinner Date: Tue, 25 Jun 2024 14:25:32 -0700 Subject: [PATCH 17/28] Missed one --- terraform/shared/container_networking/providers.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/shared/container_networking/providers.tf b/terraform/shared/container_networking/providers.tf index 01ab1f803..dec8379ee 100644 --- a/terraform/shared/container_networking/providers.tf +++ b/terraform/shared/container_networking/providers.tf @@ -1,5 +1,5 @@ terraform { - required_version = "~> 1.0" + required_version = "~> 1.7" required_providers { cloudfoundry = { source = "cloudfoundry-community/cloudfoundry" From 302a0f51f60f73943decca675dd8a9085742dcbd Mon Sep 17 00:00:00 2001 From: John Skiles Skinner Date: Tue, 25 Jun 2024 14:59:12 -0700 Subject: [PATCH 18/28] Redis v7 in sandbox and staging env main.tf --- terraform/sandbox/main.tf | 16 +++++++++++++++- terraform/staging/main.tf | 16 +++++++++++++++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/terraform/sandbox/main.tf b/terraform/sandbox/main.tf index 74c16d808..b831b44d6 100644 --- a/terraform/sandbox/main.tf +++ b/terraform/sandbox/main.tf @@ -6,7 +6,7 @@ locals { recursive_delete = true } -module "redis" { +module "redis" { # default v6.2; delete after v7.0 resource is bound source = "github.com/18f/terraform-cloudgov//redis?ref=v0.7.1" cf_org_name = local.cf_org_name @@ -16,6 +16,20 @@ module "redis" { redis_plan_name = "redis-dev" } +module "redis-v70" { + source = "github.com/GSA-TTS/terraform-cloudgov//redis?ref=v1.0.0" + + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + name = "${local.app_name}-redis-v70-${local.env}" + redis_plan_name = "redis-dev" + json_params = jsonencode( + { + "engineVersion" : "7.0", + } + ) +} + module "logo_upload_bucket" { source = "github.com/18f/terraform-cloudgov//s3?ref=v0.7.1" diff --git a/terraform/staging/main.tf b/terraform/staging/main.tf index d0df6e81d..b47e80029 100644 --- a/terraform/staging/main.tf +++ b/terraform/staging/main.tf @@ -6,7 +6,7 @@ locals { recursive_delete = true } -module "redis" { +module "redis" { # default v6.2; delete after v7.0 resource is bound source = "github.com/18f/terraform-cloudgov//redis?ref=v0.7.1" cf_org_name = local.cf_org_name @@ -16,6 +16,20 @@ module "redis" { redis_plan_name = "redis-dev" } +module "redis-v70" { + source = "github.com/GSA-TTS/terraform-cloudgov//redis?ref=v1.0.0" + + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + name = "${local.app_name}-redis-v70-${local.env}" + redis_plan_name = "redis-dev" + json_params = jsonencode( + { + "engineVersion" : "7.0", + } + ) +} + module "logo_upload_bucket" { source = "github.com/18f/terraform-cloudgov//s3?ref=v0.7.1" From 73b42b4abfefc8f3a87c152b958c4466b73f2512 Mon Sep 17 00:00:00 2001 From: John Skiles Skinner Date: Tue, 25 Jun 2024 15:05:58 -0700 Subject: [PATCH 19/28] Same new resource in Demo and Production envs --- terraform/demo/main.tf | 16 +++++++++++++++- terraform/production/main.tf | 16 +++++++++++++++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/terraform/demo/main.tf b/terraform/demo/main.tf index 7f34fb0f1..545871d4a 100644 --- a/terraform/demo/main.tf +++ b/terraform/demo/main.tf @@ -6,7 +6,7 @@ locals { recursive_delete = false } -module "redis" { +module "redis" { # default v6.2; delete after v7.0 resource is bound source = "github.com/18f/terraform-cloudgov//redis?ref=v0.7.1" cf_org_name = local.cf_org_name @@ -16,6 +16,20 @@ module "redis" { redis_plan_name = "redis-dev" } +module "redis-v70" { + source = "github.com/GSA-TTS/terraform-cloudgov//redis?ref=v1.0.0" + + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + name = "${local.app_name}-redis-v70-${local.env}" + redis_plan_name = "redis-dev" + json_params = jsonencode( + { + "engineVersion" : "7.0", + } + ) +} + module "logo_upload_bucket" { source = "github.com/18f/terraform-cloudgov//s3?ref=v0.7.1" diff --git a/terraform/production/main.tf b/terraform/production/main.tf index 450212cdf..75721dd34 100644 --- a/terraform/production/main.tf +++ b/terraform/production/main.tf @@ -6,7 +6,7 @@ locals { recursive_delete = false } -module "redis" { +module "redis" { # default v6.2; delete after v7.0 resource is bound source = "github.com/18f/terraform-cloudgov//redis?ref=v0.7.1" cf_org_name = local.cf_org_name @@ -16,6 +16,20 @@ module "redis" { redis_plan_name = "redis-3node-large" } +module "redis-v70" { + source = "github.com/GSA-TTS/terraform-cloudgov//redis?ref=v1.0.0" + + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + name = "${local.app_name}-redis-v70-${local.env}" + redis_plan_name = "redis-dev" + json_params = jsonencode( + { + "engineVersion" : "7.0", + } + ) +} + module "logo_upload_bucket" { source = "github.com/18f/terraform-cloudgov//s3?ref=v0.7.1" From f591ef21fc9186bb8baed89eb9c717e60ca7f336 Mon Sep 17 00:00:00 2001 From: John Skiles Skinner Date: Tue, 25 Jun 2024 15:19:49 -0700 Subject: [PATCH 20/28] Upgrade tj-actions from v41 to v44 --- .github/workflows/deploy-demo.yml | 4 ++-- .github/workflows/deploy-prod.yml | 4 ++-- .github/workflows/deploy.yml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/deploy-demo.yml b/.github/workflows/deploy-demo.yml index dc725f157..a8adfa918 100644 --- a/.github/workflows/deploy-demo.yml +++ b/.github/workflows/deploy-demo.yml @@ -18,7 +18,7 @@ jobs: - name: Check for changes to Terraform id: changed-terraform-files - uses: tj-actions/changed-files@v41.0.0 + uses: tj-actions/changed-files@v44 with: files: | terraform/demo @@ -88,7 +88,7 @@ jobs: - name: Check for changes to egress config id: changed-egress-config - uses: tj-actions/changed-files@v41.0.0 + uses: tj-actions/changed-files@v44 with: files: | deploy-config/egress_proxy/notify-admin-demo.*.acl diff --git a/.github/workflows/deploy-prod.yml b/.github/workflows/deploy-prod.yml index d614bf309..940e11faf 100644 --- a/.github/workflows/deploy-prod.yml +++ b/.github/workflows/deploy-prod.yml @@ -18,7 +18,7 @@ jobs: - name: Check for changes to Terraform id: changed-terraform-files - uses: tj-actions/changed-files@v41.0.0 + uses: tj-actions/changed-files@v44 with: files: | terraform/production @@ -88,7 +88,7 @@ jobs: - name: Check for changes to egress config id: changed-egress-config - uses: tj-actions/changed-files@v41.0.0 + uses: tj-actions/changed-files@v44 with: files: | deploy-config/egress_proxy/notify-admin-production.*.acl diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index d74ba3133..f3b6133a4 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -23,7 +23,7 @@ jobs: - name: Check for changes to Terraform id: changed-terraform-files - uses: tj-actions/changed-files@v41.0.0 + uses: tj-actions/changed-files@v44 with: files: | terraform/staging @@ -95,7 +95,7 @@ jobs: - name: Check for changes to egress config id: changed-egress-config - uses: tj-actions/changed-files@v41.0.0 + uses: tj-actions/changed-files@v44 with: files: | deploy-config/egress_proxy/notify-admin-staging.*.acl From f218c0124436ba6f081726e5290e550156467055 Mon Sep 17 00:00:00 2001 From: Andrew Shumway Date: Wed, 26 Jun 2024 08:34:05 -0600 Subject: [PATCH 21/28] Remove commented out code --- app/templates/views/notifications.html | 4 ---- 1 file changed, 4 deletions(-) diff --git a/app/templates/views/notifications.html b/app/templates/views/notifications.html index 487e10b98..cac2b9811 100644 --- a/app/templates/views/notifications.html +++ b/app/templates/views/notifications.html @@ -76,10 +76,6 @@ Download all data last 3 days (CSV)  

-

Download all data today (CSV)   From 18da813fe65b9c7951f791410c3f67bbf90c450a Mon Sep 17 00:00:00 2001 From: alexjanousekGSA Date: Wed, 26 Jun 2024 11:08:20 -0600 Subject: [PATCH 22/28] Updated readme --- README.md | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 1c4df2aa5..15a86ae00 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ You will need the following items: [Follow the instructions here to set up the Notify.gov API.](https://github.com/GSA/notifications-api#before-you-start) The Notify.gov API is required in order for the Notify.gov Admin UI to run, and -it will also take care of many of the steps that are listed here. The sections +it will also take care of many of the steps that are listed here. The sections that are a repeat from the API setup are flagged with an **[API Step]** label in front of them. @@ -83,11 +83,13 @@ Your system `$PATH` environment variable is likely set in one of these locations: For BASH shells: + - `~/.bashrc` - `~/.bash_profile` - `~/.profile` For ZSH shells: + - `~/.zshrc` - `~/.zprofile` @@ -97,7 +99,7 @@ environments. Which file you need to modify depends on whether or not you are running an interactive shell or a login shell (see [this Stack Overflow post](https://stackoverflow.com/questions/18186929/what-are-the-differences-between-a-login-shell-and-interactive-shell) -for an explanation of the differences). If you're still not sure, please ask +for an explanation of the differences). If you're still not sure, please ask the team for help! Once you determine which file you'll need to modify, add these lines before any @@ -158,7 +160,7 @@ _NOTE: This project currently uses the latest `1.4.x release of Terraform._ #### [API Step] Python Installation Now we're going to install a tool to help us manage Python versions and -virtual environments on our system. First, we'll install +virtual environments on our system. First, we'll install [pyenv](https://github.com/pyenv/pyenv) and one of its plugins, [pyenv-virtualenv](https://github.com/pyenv/pyenv-virtualenv), with Homebrew: @@ -285,7 +287,7 @@ we'll use `3.12` in our example here since we recently upgraded to this version: pyenv install 3.12 ``` -Next, delete the virtual environment you previously had set up. If you followed +Next, delete the virtual environment you previously had set up. If you followed the instructions above with the first-time set up, you can do this with `pyenv`: ```sh @@ -306,6 +308,20 @@ you'll be set with an upgraded version of Python. _If you're not sure about the details of your current virtual environment, you can run `poetry env info` to get more information. If you've been using `pyenv` for everything, you can also see all available virtual environments with `pyenv virtualenvs`._ +#### Updating the .env file for Login.gov + +To configure the application for Login.gov, you will need to update the following environment variables in the .env file: + +``` +COMMIT_HASH=”--------” +``` + +Reach out to someone on the team to get the most recent Login.gov key. + +``` +LOGIN_PEM="INSERT_LOGIN_GOV_KEY_HERE" +``` + #### Updating the .env file for E2E tests With the newly created `.env` file in place, you'll need to make one more @@ -353,7 +369,7 @@ API is running as well! ## Creating a 'First User' in the database After you have completed all setup steps, you will be unable to log in, because there -will not be a user in the database to link to the login.gov account you are using. So +will not be a user in the database to link to the login.gov account you are using. So you will need to create that user in your database using the 'create-test-user' command. Open two terminals pointing to the api project and then run these commands in the @@ -372,8 +388,6 @@ is the same one you are using in login.gov and make sure your phone number is in If for any reason in the course of development it is necessary for your to delete your db via the `dropdb` command, you will need to repeat these steps when you recreate your db. - - ## Git Hooks We're using [`pre-commit`](https://pre-commit.com/) to manage hooks in order to From 41d1a3e1b580628b581199cf272020bd3ae78892 Mon Sep 17 00:00:00 2001 From: John Skiles Skinner Date: Wed, 26 Jun 2024 14:51:20 -0700 Subject: [PATCH 23/28] Update name of Redis to include v70 --- manifest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifest.yml b/manifest.yml index ed596ffeb..bccd00db6 100644 --- a/manifest.yml +++ b/manifest.yml @@ -12,7 +12,7 @@ applications: - route: ((cloud_dot_gov_route)) services: - - notify-admin-redis-((env)) + - notify-admin-redis-v70-((env)) - notify-api-csv-upload-bucket-((env)) - notify-admin-logo-upload-bucket-((env)) From 789ca4cb1bf18e47d3938b84fa08d355d09a0142 Mon Sep 17 00:00:00 2001 From: John Skiles Skinner Date: Wed, 26 Jun 2024 15:16:15 -0700 Subject: [PATCH 24/28] Added null_resource technique previously used in API repo --- terraform/staging/main.tf | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/terraform/staging/main.tf b/terraform/staging/main.tf index b47e80029..0cc72358a 100644 --- a/terraform/staging/main.tf +++ b/terraform/staging/main.tf @@ -6,6 +6,14 @@ locals { recursive_delete = true } +resource "null_resource" "prevent_destroy" { + + lifecycle { + prevent_destroy = false # destroying staging is allowed + } +} + + module "redis" { # default v6.2; delete after v7.0 resource is bound source = "github.com/18f/terraform-cloudgov//redis?ref=v0.7.1" From ece3bd201d75116e07b188e6ce4f2e922499b39f Mon Sep 17 00:00:00 2001 From: John Skiles Skinner Date: Wed, 26 Jun 2024 15:44:01 -0700 Subject: [PATCH 25/28] Adjust indentation in deploy.yml --- .github/workflows/deploy.yml | 166 +++++++++++++++++------------------ 1 file changed, 83 insertions(+), 83 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index f3b6133a4..e401e8ee0 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -17,96 +17,96 @@ jobs: environment: staging steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 2 + - uses: actions/checkout@v4 + with: + fetch-depth: 2 - - name: Check for changes to Terraform - id: changed-terraform-files - uses: tj-actions/changed-files@v44 - with: - files: | - terraform/staging - terraform/shared - .github/workflows/deploy.yml - - name: Terraform init - if: steps.changed-terraform-files.outputs.any_changed == 'true' - working-directory: terraform/staging - env: - AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }} - run: terraform init - - name: Terraform apply - if: steps.changed-terraform-files.outputs.any_changed == 'true' - working-directory: terraform/staging - env: - AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }} - TF_VAR_cf_user: ${{ secrets.CLOUDGOV_USERNAME }} - TF_VAR_cf_password: ${{ secrets.CLOUDGOV_PASSWORD }} - run: terraform apply -auto-approve -input=false + - name: Check for changes to Terraform + id: changed-terraform-files + uses: tj-actions/changed-files@v44 + with: + files: | + terraform/staging + terraform/shared + .github/workflows/deploy.yml + - name: Terraform init + if: steps.changed-terraform-files.outputs.any_changed == 'true' + working-directory: terraform/staging + env: + AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }} + run: terraform init + - name: Terraform apply + if: steps.changed-terraform-files.outputs.any_changed == 'true' + working-directory: terraform/staging + env: + AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }} + TF_VAR_cf_user: ${{ secrets.CLOUDGOV_USERNAME }} + TF_VAR_cf_password: ${{ secrets.CLOUDGOV_PASSWORD }} + run: terraform apply -auto-approve -input=false - - uses: ./.github/actions/setup-project + - uses: ./.github/actions/setup-project - - name: Create requirements.txt - run: poetry export --without-hashes --format=requirements.txt > requirements.txt + - name: Create requirements.txt + run: poetry export --without-hashes --format=requirements.txt > requirements.txt - - name: Deploy to cloud.gov - uses: 18f/cg-deploy-action@main - env: - DANGEROUS_SALT: ${{ secrets.DANGEROUS_SALT }} - SECRET_KEY: ${{ secrets.SECRET_KEY }} - ADMIN_CLIENT_SECRET: ${{ secrets.ADMIN_CLIENT_SECRET }} - NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }} - NR_BROWSER_KEY: ${{ secrets.NR_BROWSER_KEY }} - COMMIT_HASH: ${{ github.sha }} - LOGIN_PEM: ${{ secrets.LOGIN_PEM }} - LOGIN_DOT_GOV_CLIENT_ID: "urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov" - LOGIN_DOT_GOV_USER_INFO_URL: "https://secure.login.gov/api/openid_connect/userinfo" - LOGIN_DOT_GOV_ACCESS_TOKEN_URL: "https://secure.login.gov/api/openid_connect/token" - LOGIN_DOT_GOV_LOGOUT_URL: "https://secure.login.gov/openid_connect/logout?client_id=urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov&post_logout_redirect_uri=https://notify-staging.app.cloud.gov/sign-out" - LOGIN_DOT_GOV_BASE_LOGOUT_URL: "https://secure.login.gov/openid_connect/logout?" - LOGIN_DOT_GOV_SIGNOUT_REDIRECT: "https://notify-staging.app.cloud.gov/sign-out" - LOGIN_DOT_GOV_INITIAL_SIGNIN_URL: "https://secure.login.gov/openid_connect/authorize?acr_values=http%3A%2F%2Fidmanagement.gov%2Fns%2Fassurance%2Fial%2F1&client_id=urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov&nonce=NONCE&prompt=select_account&redirect_uri=https://notify-staging.app.cloud.gov/sign-in&response_type=code&scope=openid+email&state=STATEE" - with: - cf_username: ${{ secrets.CLOUDGOV_USERNAME }} - cf_password: ${{ secrets.CLOUDGOV_PASSWORD }} - cf_org: gsa-tts-benefits-studio - cf_space: notify-staging - push_arguments: >- - --vars-file deploy-config/staging.yml - --var DANGEROUS_SALT="$DANGEROUS_SALT" - --var SECRET_KEY="$SECRET_KEY" - --var ADMIN_CLIENT_USERNAME="notify-admin" - --var ADMIN_CLIENT_SECRET="$ADMIN_CLIENT_SECRET" - --var NEW_RELIC_LICENSE_KEY="$NEW_RELIC_LICENSE_KEY" - --var NR_BROWSER_KEY="$NR_BROWSER_KEY" - --var COMMIT_HASH="$COMMIT_HASH" - --var LOGIN_PEM="$LOGIN_PEM" - --var LOGIN_DOT_GOV_CLIENT_ID="$LOGIN_DOT_GOV_CLIENT_ID" - --var LOGIN_DOT_GOV_USER_INFO_URL="$LOGIN_DOT_GOV_USER_INFO_URL" - --var LOGIN_DOT_GOV_ACCESS_TOKEN_URL="$LOGIN_DOT_GOV_ACCESS_TOKEN_URL" - --var LOGIN_DOT_GOV_LOGOUT_URL="$LOGIN_DOT_GOV_LOGOUT_URL" - --var LOGIN_DOT_GOV_BASE_LOGOUT_URL="$LOGIN_DOT_GOV_BASE_LOGOUT_URL" - --var LOGIN_DOT_GOV_SIGNOUT_REDIRECT="$LOGIN_DOT_GOV_SIGNOUT_REDIRECT" - --var LOGIN_DOT_GOV_INITIAL_SIGNIN_URL="$LOGIN_DOT_GOV_INITIAL_SIGNIN_URL" + - name: Deploy to cloud.gov + uses: 18f/cg-deploy-action@main + env: + DANGEROUS_SALT: ${{ secrets.DANGEROUS_SALT }} + SECRET_KEY: ${{ secrets.SECRET_KEY }} + ADMIN_CLIENT_SECRET: ${{ secrets.ADMIN_CLIENT_SECRET }} + NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }} + NR_BROWSER_KEY: ${{ secrets.NR_BROWSER_KEY }} + COMMIT_HASH: ${{ github.sha }} + LOGIN_PEM: ${{ secrets.LOGIN_PEM }} + LOGIN_DOT_GOV_CLIENT_ID: "urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov" + LOGIN_DOT_GOV_USER_INFO_URL: "https://secure.login.gov/api/openid_connect/userinfo" + LOGIN_DOT_GOV_ACCESS_TOKEN_URL: "https://secure.login.gov/api/openid_connect/token" + LOGIN_DOT_GOV_LOGOUT_URL: "https://secure.login.gov/openid_connect/logout?client_id=urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov&post_logout_redirect_uri=https://notify-staging.app.cloud.gov/sign-out" + LOGIN_DOT_GOV_BASE_LOGOUT_URL: "https://secure.login.gov/openid_connect/logout?" + LOGIN_DOT_GOV_SIGNOUT_REDIRECT: "https://notify-staging.app.cloud.gov/sign-out" + LOGIN_DOT_GOV_INITIAL_SIGNIN_URL: "https://secure.login.gov/openid_connect/authorize?acr_values=http%3A%2F%2Fidmanagement.gov%2Fns%2Fassurance%2Fial%2F1&client_id=urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov&nonce=NONCE&prompt=select_account&redirect_uri=https://notify-staging.app.cloud.gov/sign-in&response_type=code&scope=openid+email&state=STATEE" + with: + cf_username: ${{ secrets.CLOUDGOV_USERNAME }} + cf_password: ${{ secrets.CLOUDGOV_PASSWORD }} + cf_org: gsa-tts-benefits-studio + cf_space: notify-staging + push_arguments: >- + --vars-file deploy-config/staging.yml + --var DANGEROUS_SALT="$DANGEROUS_SALT" + --var SECRET_KEY="$SECRET_KEY" + --var ADMIN_CLIENT_USERNAME="notify-admin" + --var ADMIN_CLIENT_SECRET="$ADMIN_CLIENT_SECRET" + --var NEW_RELIC_LICENSE_KEY="$NEW_RELIC_LICENSE_KEY" + --var NR_BROWSER_KEY="$NR_BROWSER_KEY" + --var COMMIT_HASH="$COMMIT_HASH" + --var LOGIN_PEM="$LOGIN_PEM" + --var LOGIN_DOT_GOV_CLIENT_ID="$LOGIN_DOT_GOV_CLIENT_ID" + --var LOGIN_DOT_GOV_USER_INFO_URL="$LOGIN_DOT_GOV_USER_INFO_URL" + --var LOGIN_DOT_GOV_ACCESS_TOKEN_URL="$LOGIN_DOT_GOV_ACCESS_TOKEN_URL" + --var LOGIN_DOT_GOV_LOGOUT_URL="$LOGIN_DOT_GOV_LOGOUT_URL" + --var LOGIN_DOT_GOV_BASE_LOGOUT_URL="$LOGIN_DOT_GOV_BASE_LOGOUT_URL" + --var LOGIN_DOT_GOV_SIGNOUT_REDIRECT="$LOGIN_DOT_GOV_SIGNOUT_REDIRECT" + --var LOGIN_DOT_GOV_INITIAL_SIGNIN_URL="$LOGIN_DOT_GOV_INITIAL_SIGNIN_URL" - - name: Check for changes to egress config - id: changed-egress-config - uses: tj-actions/changed-files@v44 - with: - files: | - deploy-config/egress_proxy/notify-admin-staging.*.acl - .github/actions/deploy-proxy/action.yml - .github/workflows/deploy.yml - - name: Deploy egress proxy - if: steps.changed-egress-config.outputs.any_changed == 'true' - uses: ./.github/actions/deploy-proxy - with: - cf_space: notify-staging - app: notify-admin-staging + - name: Check for changes to egress config + id: changed-egress-config + uses: tj-actions/changed-files@v44 + with: + files: | + deploy-config/egress_proxy/notify-admin-staging.*.acl + .github/actions/deploy-proxy/action.yml + .github/workflows/deploy.yml + - name: Deploy egress proxy + if: steps.changed-egress-config.outputs.any_changed == 'true' + uses: ./.github/actions/deploy-proxy + with: + cf_space: notify-staging + app: notify-admin-staging bail: runs-on: ubuntu-latest From 3e258d3910f5f55c36fd96a708107cd704bec4da Mon Sep 17 00:00:00 2001 From: John Skiles Skinner Date: Wed, 26 Jun 2024 18:13:13 -0700 Subject: [PATCH 26/28] Temp remove new Redis, revert to old in Manifest --- manifest.yml | 2 +- terraform/production/main.tf | 14 -------------- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/manifest.yml b/manifest.yml index bccd00db6..ed596ffeb 100644 --- a/manifest.yml +++ b/manifest.yml @@ -12,7 +12,7 @@ applications: - route: ((cloud_dot_gov_route)) services: - - notify-admin-redis-v70-((env)) + - notify-admin-redis-((env)) - notify-api-csv-upload-bucket-((env)) - notify-admin-logo-upload-bucket-((env)) diff --git a/terraform/production/main.tf b/terraform/production/main.tf index 75721dd34..be4bf35ea 100644 --- a/terraform/production/main.tf +++ b/terraform/production/main.tf @@ -16,20 +16,6 @@ module "redis" { # default v6.2; delete after v7.0 resource is bound redis_plan_name = "redis-3node-large" } -module "redis-v70" { - source = "github.com/GSA-TTS/terraform-cloudgov//redis?ref=v1.0.0" - - cf_org_name = local.cf_org_name - cf_space_name = local.cf_space_name - name = "${local.app_name}-redis-v70-${local.env}" - redis_plan_name = "redis-dev" - json_params = jsonencode( - { - "engineVersion" : "7.0", - } - ) -} - module "logo_upload_bucket" { source = "github.com/18f/terraform-cloudgov//s3?ref=v0.7.1" From ea1dc14ab3cd68f3e25b7e60abf0e7a32980c38c Mon Sep 17 00:00:00 2001 From: John Skiles Skinner Date: Wed, 26 Jun 2024 18:47:21 -0700 Subject: [PATCH 27/28] Add /** to paths in three deploy YAML files --- .github/workflows/deploy-demo.yml | 4 ++-- .github/workflows/deploy-prod.yml | 4 ++-- .github/workflows/deploy.yml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/deploy-demo.yml b/.github/workflows/deploy-demo.yml index a8adfa918..89adc1f29 100644 --- a/.github/workflows/deploy-demo.yml +++ b/.github/workflows/deploy-demo.yml @@ -21,8 +21,8 @@ jobs: uses: tj-actions/changed-files@v44 with: files: | - terraform/demo - terraform/shared + terraform/demo/** + terraform/shared/** .github/workflows/deploy-demo.yml - name: Terraform init if: steps.changed-terraform-files.outputs.any_changed == 'true' diff --git a/.github/workflows/deploy-prod.yml b/.github/workflows/deploy-prod.yml index 940e11faf..262079be8 100644 --- a/.github/workflows/deploy-prod.yml +++ b/.github/workflows/deploy-prod.yml @@ -21,8 +21,8 @@ jobs: uses: tj-actions/changed-files@v44 with: files: | - terraform/production - terraform/shared + terraform/production/** + terraform/shared/** .github/workflows/deploy-prod.yml - name: Terraform init if: steps.changed-terraform-files.outputs.any_changed == 'true' diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index e401e8ee0..8cf33babc 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -26,8 +26,8 @@ jobs: uses: tj-actions/changed-files@v44 with: files: | - terraform/staging - terraform/shared + terraform/staging/** + terraform/shared/** .github/workflows/deploy.yml - name: Terraform init if: steps.changed-terraform-files.outputs.any_changed == 'true' From e897800cb5d2cea45069d1b3794b38c54f425e6f Mon Sep 17 00:00:00 2001 From: John Skiles Skinner Date: Wed, 26 Jun 2024 19:11:55 -0700 Subject: [PATCH 28/28] Restore v7.0 Redis at the correct plan name in Prod --- manifest.yml | 2 +- terraform/production/main.tf | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/manifest.yml b/manifest.yml index ed596ffeb..bccd00db6 100644 --- a/manifest.yml +++ b/manifest.yml @@ -12,7 +12,7 @@ applications: - route: ((cloud_dot_gov_route)) services: - - notify-admin-redis-((env)) + - notify-admin-redis-v70-((env)) - notify-api-csv-upload-bucket-((env)) - notify-admin-logo-upload-bucket-((env)) diff --git a/terraform/production/main.tf b/terraform/production/main.tf index be4bf35ea..69cc9b264 100644 --- a/terraform/production/main.tf +++ b/terraform/production/main.tf @@ -16,6 +16,20 @@ module "redis" { # default v6.2; delete after v7.0 resource is bound redis_plan_name = "redis-3node-large" } +module "redis-v70" { + source = "github.com/GSA-TTS/terraform-cloudgov//redis?ref=v1.0.0" + + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + name = "${local.app_name}-redis-v70-${local.env}" + redis_plan_name = "redis-3node-large" + json_params = jsonencode( + { + "engineVersion" : "7.0", + } + ) +} + module "logo_upload_bucket" { source = "github.com/18f/terraform-cloudgov//s3?ref=v0.7.1"