From 76c34ffae68feb2b7ea41d719eca8aebe4544239 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Tue, 11 Jun 2024 10:34:57 -0700 Subject: [PATCH 01/28] Need magic PII-free debugging method for API --- app/celery/tasks.py | 10 +++++--- app/clients/cloudwatch/aws_cloudwatch.py | 6 ++--- app/delivery/send_to_providers.py | 6 ++--- app/notifications/process_notifications.py | 6 +++-- app/utils.py | 13 ---------- notifications_utils/logging.py | 29 ++++++++++++++++++++++ poetry.lock | 8 ++---- tests/app/test_utils.py | 8 ------ tests/notifications_utils/test_logging.py | 17 +++++++++++++ 9 files changed, 65 insertions(+), 38 deletions(-) diff --git a/app/celery/tasks.py b/app/celery/tasks.py index 1950712af..c59c40b6a 100644 --- a/app/celery/tasks.py +++ b/app/celery/tasks.py @@ -23,7 +23,7 @@ from app.notifications.process_notifications import persist_notification from app.notifications.validators import check_service_over_total_message_limit from app.serialised_models import SerialisedService, SerialisedTemplate from app.service.utils import service_allowed_to_send_to -from app.utils import DATETIME_FORMAT, hilite, scrub, utc_now +from app.utils import DATETIME_FORMAT, hilite, utc_now from app.v2.errors import TotalRequestsError from notifications_utils.recipients import RecipientCSV @@ -189,7 +189,9 @@ def save_sms(self, service_id, notification_id, encrypted_notification, sender_i # Return False when trial mode services try sending notifications # to non-team and non-simulated recipients. if not service_allowed_to_send_to(notification["to"], service, KeyType.NORMAL): - current_app.logger.info(hilite(scrub(f"service not allowed to send to {notification['to']}, aborting"))) + current_app.logger.info( + hilite(f"service not allowed to send to {notification['to']}, aborting") + ) current_app.logger.debug( "SMS {} failed as restricted service".format(notification_id) ) @@ -220,7 +222,9 @@ def save_sms(self, service_id, notification_id, encrypted_notification, sender_i ) # Kick off sns process in provider_tasks.py - current_app.logger.info(hilite(scrub(f"Going to deliver sms for recipient: {notification['to']}"))) + current_app.logger.info( + hilite(f"Going to deliver sms for recipient: {notification['to']}") + ) provider_tasks.deliver_sms.apply_async( [str(saved_notification.id)], queue=QueueNames.SEND_SMS ) diff --git a/app/clients/cloudwatch/aws_cloudwatch.py b/app/clients/cloudwatch/aws_cloudwatch.py index 14d8513c5..624a49820 100644 --- a/app/clients/cloudwatch/aws_cloudwatch.py +++ b/app/clients/cloudwatch/aws_cloudwatch.py @@ -9,7 +9,7 @@ from flask import current_app from app.clients import AWS_CLIENT_CONFIG, Client from app.cloudfoundry_config import cloud_config from app.exceptions import NotificationTechnicalFailureException -from app.utils import hilite, scrub, utc_now +from app.utils import hilite, utc_now class AwsCloudwatchClient(Client): @@ -124,7 +124,7 @@ class AwsCloudwatchClient(Client): self.warn_if_dev_is_opted_out( message["delivery"]["providerResponse"], notification_id ) - current_app.logger.info(hilite(scrub(f"DELIVERED: {message}"))) + current_app.logger.info(hilite(f"DELIVERED: {message}")) return ( "success", message["delivery"]["providerResponse"], @@ -142,7 +142,7 @@ class AwsCloudwatchClient(Client): message["delivery"]["providerResponse"], notification_id ) - current_app.logger.info(hilite(scrub(f"FAILED: {message}"))) + current_app.logger.info(hilite(f"FAILED: {message}")) return ( "failure", message["delivery"]["providerResponse"], diff --git a/app/delivery/send_to_providers.py b/app/delivery/send_to_providers.py index a347b89ad..17b0ea134 100644 --- a/app/delivery/send_to_providers.py +++ b/app/delivery/send_to_providers.py @@ -13,7 +13,7 @@ from app.dao.provider_details_dao import get_provider_details_by_notification_ty from app.enums import BrandType, KeyType, NotificationStatus, NotificationType from app.exceptions import NotificationTechnicalFailureException from app.serialised_models import SerialisedService, SerialisedTemplate -from app.utils import hilite, scrub, utc_now +from app.utils import hilite, utc_now from notifications_utils.template import ( HTMLEmailTemplate, PlainTextEmailTemplate, @@ -114,14 +114,14 @@ def send_sms_to_provider(notification): current_app.logger.info(f"got message_id {message_id}") except Exception as e: msg = f"FAILED sending message for this recipient: {recipient} to sms" - current_app.logger.error(hilite(scrub(f"{msg} {e}"))) + current_app.logger.error(hilite(f"{msg} {e}")) notification.billable_units = template.fragment_count dao_update_notification(notification) raise e else: msg = f"Sending message for this recipient: {recipient} to sms" - current_app.logger.info(hilite(scrub(msg))) + current_app.logger.info(hilite(msg)) notification.billable_units = template.fragment_count update_notification_to_sending(notification, provider) return message_id diff --git a/app/notifications/process_notifications.py b/app/notifications/process_notifications.py index d899a8146..a197467d1 100644 --- a/app/notifications/process_notifications.py +++ b/app/notifications/process_notifications.py @@ -11,7 +11,7 @@ from app.dao.notifications_dao import ( ) from app.enums import KeyType, NotificationStatus, NotificationType from app.models import Notification -from app.utils import hilite, scrub, utc_now +from app.utils import hilite, utc_now from app.v2.errors import BadRequestError from notifications_utils.recipients import ( format_email_address, @@ -110,7 +110,9 @@ def persist_notification( formatted_recipient = validate_and_format_phone_number( recipient, international=True ) - current_app.logger.info(hilite(scrub(f"Persisting notification with recipient {formatted_recipient}"))) + current_app.logger.info( + hilite(f"Persisting notification with recipient {formatted_recipient}") + ) recipient_info = get_international_phone_info(formatted_recipient) notification.normalised_to = formatted_recipient notification.international = recipient_info.international diff --git a/app/utils.py b/app/utils.py index b23a000d1..e286d6539 100644 --- a/app/utils.py +++ b/app/utils.py @@ -1,4 +1,3 @@ -import re from datetime import datetime, timedelta, timezone from flask import url_for @@ -145,15 +144,3 @@ def naive_utcnow(): def utc_now(): return naive_utcnow() - - -def scrub(msg): - # Eventually we want to scrub all messages in all logs for phone numbers - # and email addresses, masking them. Ultimately this will probably get - # refactored into a 'SafeLogger' subclass or something, but let's start here - # with phones. - phones = re.findall("(?:\\+ *)?\\d[\\d\\- ]{7,}\\d", msg) - phones = [phone.replace("-", "").replace(" ", "") for phone in phones] - for phone in phones: - msg = msg.replace(phone, f"1XXXXX{phone[-5:]}") - return msg diff --git a/notifications_utils/logging.py b/notifications_utils/logging.py index 6a209cdd3..750fcfc48 100644 --- a/notifications_utils/logging.py +++ b/notifications_utils/logging.py @@ -1,5 +1,6 @@ import logging import logging.handlers +import re import sys from itertools import product @@ -131,3 +132,31 @@ class JSONFormatter(BaseJSONFormatter): except (KeyError, IndexError) as e: logger.exception("failed to format log message: {} not found".format(e)) return log_record + + +class PIIFormatter(BaseJSONFormatter): + + def scrub(self, msg): + # Eventually we want to scrub all messages in all logs for phone numbers + # and email addresses, masking them. Ultimately this will probably get + # refactored into a 'SafeLogger' subclass or something, but let's start here + # with phones. + phones = re.findall("(?:\\+ *)?\\d[\\d\\- ]{7,}\\d", msg) + phones = [phone.replace("-", "").replace(" ", "") for phone in phones] + for phone in phones: + msg = msg.replace(phone, f"1XXXXX{phone[-5:]}") + + emails = re.findall( + r"[\w\.-]+@[\w\.-]+", msg + ) # ['alice@google.com', 'bob@abc.com'] + for email in emails: + # do something with each found email string + msg = msg.replace(email, f"XXXXX{email[-10:]}") + return msg + + def process_log_record(self, log_record): + try: + log_record["message"] = self.scrub(log_record["message"]) + except (KeyError, IndexError, ValueError) as e: + logger.exception(f"failed to scrub log message: {e}") + return log_record diff --git a/poetry.lock b/poetry.lock index 2ee7144b2..b77891a4a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. [[package]] name = "aiohttp" @@ -2098,13 +2098,9 @@ files = [ {file = "lxml-5.2.2-cp36-cp36m-win_amd64.whl", hash = "sha256:edcfa83e03370032a489430215c1e7783128808fd3e2e0a3225deee278585196"}, {file = "lxml-5.2.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:28bf95177400066596cdbcfc933312493799382879da504633d16cf60bba735b"}, {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3a745cc98d504d5bd2c19b10c79c61c7c3df9222629f1b6210c0368177589fb8"}, - {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1b590b39ef90c6b22ec0be925b211298e810b4856909c8ca60d27ffbca6c12e6"}, {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b336b0416828022bfd5a2e3083e7f5ba54b96242159f83c7e3eebaec752f1716"}, - {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_28_aarch64.whl", hash = "sha256:c2faf60c583af0d135e853c86ac2735ce178f0e338a3c7f9ae8f622fd2eb788c"}, {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:4bc6cb140a7a0ad1f7bc37e018d0ed690b7b6520ade518285dc3171f7a117905"}, - {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7ff762670cada8e05b32bf1e4dc50b140790909caa8303cfddc4d702b71ea184"}, {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:57f0a0bbc9868e10ebe874e9f129d2917750adf008fe7b9c1598c0fbbfdde6a6"}, - {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:a6d2092797b388342c1bc932077ad232f914351932353e2e8706851c870bca1f"}, {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:60499fe961b21264e17a471ec296dcbf4365fbea611bf9e303ab69db7159ce61"}, {file = "lxml-5.2.2-cp37-cp37m-win32.whl", hash = "sha256:d9b342c76003c6b9336a80efcc766748a333573abf9350f4094ee46b006ec18f"}, {file = "lxml-5.2.2-cp37-cp37m-win_amd64.whl", hash = "sha256:b16db2770517b8799c79aa80f4053cd6f8b716f21f8aca962725a9565ce3ee40"}, @@ -2493,6 +2489,7 @@ files = [ {file = "msgpack-1.0.8-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5fbb160554e319f7b22ecf530a80a3ff496d38e8e07ae763b9e82fadfe96f273"}, {file = "msgpack-1.0.8-cp39-cp39-win32.whl", hash = "sha256:f9af38a89b6a5c04b7d18c492c8ccf2aee7048aff1ce8437c4683bb5a1df893d"}, {file = "msgpack-1.0.8-cp39-cp39-win_amd64.whl", hash = "sha256:ed59dd52075f8fc91da6053b12e8c89e37aa043f8986efd89e61fae69dc1b011"}, + {file = "msgpack-1.0.8-py3-none-any.whl", hash = "sha256:24f727df1e20b9876fa6e95f840a2a2651e34c0ad147676356f4bf5fbb0206ca"}, {file = "msgpack-1.0.8.tar.gz", hash = "sha256:95c02b0e27e706e48d0e5426d1710ca78e0f0628d6e89d5b5a5b91a5f12274f3"}, ] @@ -3475,7 +3472,6 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, diff --git a/tests/app/test_utils.py b/tests/app/test_utils.py index bbe37256a..b39d3a2c0 100644 --- a/tests/app/test_utils.py +++ b/tests/app/test_utils.py @@ -13,7 +13,6 @@ from app.utils import ( get_reference_from_personalisation, get_uuid_string_or_none, midnight_n_days_ago, - scrub, ) @@ -95,13 +94,6 @@ def test_get_public_notify_type_text(): ) -def test_scrub(): - result = scrub( - "This is a message with 17775554324, and also 18884449323 and also 17775554324" - ) - assert result == "This is a message with 1XXXXX54324, and also 1XXXXX49323 and also 1XXXXX54324" - - # This method is used for simulating bulk sends. We use localstack and run on a developer's machine to do the # simulation. Please see docs->bulk_testing.md for instructions. # def test_generate_csv_for_bulk_testing(): diff --git a/tests/notifications_utils/test_logging.py b/tests/notifications_utils/test_logging.py index 1aefb4065..6746a1e13 100644 --- a/tests/notifications_utils/test_logging.py +++ b/tests/notifications_utils/test_logging.py @@ -49,3 +49,20 @@ def test_base_json_formatter_contains_service_id(): == "message to log" ) assert service_id_filter.filter(record).service_id == "notify-admin" + + +def test_scrub(): + record = builtin_logging.LogRecord( + name="log thing", + level="info", + pathname="path", + lineno=123, + msg="phone1: 1555555555, phone2: 1555555554, email1: fake@fake.gov, email2: fake@fake2.fake.gov", + exc_info=None, + args=None, + ) + + assert ( + json.loads(logging.PIIFormatter().format(record))["message"] + == "phone1: 1XXXXX55555, phone2: 1XXXXX55554, email1: XXXXXe@fake.gov, email2: XXXXX2.fake.gov" + ) From bfae1f57f2b98ca5f2f9ea70e8a4554952d69a4d Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Tue, 11 Jun 2024 12:17:14 -0700 Subject: [PATCH 02/28] hmmm --- notifications_utils/logging.py | 43 +++++++++++------------ tests/notifications_utils/test_logging.py | 10 +++--- 2 files changed, 24 insertions(+), 29 deletions(-) diff --git a/notifications_utils/logging.py b/notifications_utils/logging.py index 750fcfc48..f224533af 100644 --- a/notifications_utils/logging.py +++ b/notifications_utils/logging.py @@ -70,6 +70,7 @@ def configure_handler(handler, app, formatter): handler.addFilter(AppNameFilter(app.config["NOTIFY_APP_NAME"])) handler.addFilter(RequestIdFilter()) handler.addFilter(ServiceIdFilter()) + handler.addFilter(PIIFilter()) return handler @@ -116,26 +117,7 @@ class ServiceIdFilter(logging.Filter): return record -class JSONFormatter(BaseJSONFormatter): - def process_log_record(self, log_record): - rename_map = { - "asctime": "time", - "request_id": "requestId", - "app_name": "application", - "service_id": "service_id", - } - for key, newkey in rename_map.items(): - log_record[newkey] = log_record.pop(key) - log_record["logType"] = "application" - try: - log_record["message"] = log_record["message"].format(**log_record) - except (KeyError, IndexError) as e: - logger.exception("failed to format log message: {} not found".format(e)) - return log_record - - -class PIIFormatter(BaseJSONFormatter): - +class PIIFilter(logging.Filter): def scrub(self, msg): # Eventually we want to scrub all messages in all logs for phone numbers # and email addresses, masking them. Ultimately this will probably get @@ -154,9 +136,24 @@ class PIIFormatter(BaseJSONFormatter): msg = msg.replace(email, f"XXXXX{email[-10:]}") return msg + def filter(self, record): + record.msg = self.scrub(record.msg) + return record + + +class JSONFormatter(BaseJSONFormatter): def process_log_record(self, log_record): + rename_map = { + "asctime": "time", + "request_id": "requestId", + "app_name": "application", + "service_id": "service_id", + } + for key, newkey in rename_map.items(): + log_record[newkey] = log_record.pop(key) + log_record["logType"] = "application" try: - log_record["message"] = self.scrub(log_record["message"]) - except (KeyError, IndexError, ValueError) as e: - logger.exception(f"failed to scrub log message: {e}") + log_record["message"] = log_record["message"].format(**log_record) + except (KeyError, IndexError) as e: + logger.exception("failed to format log message: {} not found".format(e)) return log_record diff --git a/tests/notifications_utils/test_logging.py b/tests/notifications_utils/test_logging.py index 6746a1e13..8f4e53923 100644 --- a/tests/notifications_utils/test_logging.py +++ b/tests/notifications_utils/test_logging.py @@ -51,7 +51,7 @@ def test_base_json_formatter_contains_service_id(): assert service_id_filter.filter(record).service_id == "notify-admin" -def test_scrub(): +def test_pii_filter(): record = builtin_logging.LogRecord( name="log thing", level="info", @@ -61,8 +61,6 @@ def test_scrub(): exc_info=None, args=None, ) - - assert ( - json.loads(logging.PIIFormatter().format(record))["message"] - == "phone1: 1XXXXX55555, phone2: 1XXXXX55554, email1: XXXXXe@fake.gov, email2: XXXXX2.fake.gov" - ) + pii_filter = logging.PIIFilter() + x = pii_filter.filter(record) + assert pii_filter.filter(record).msg == "phone1: 1XXXXX55555, phone2: 1XXXXX55554, email1: XXXXXe@fake.gov, email2: XXXXX2.fake.gov" From d1ab771756805de9c06e2f2ebe02776b70f0c77c Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Tue, 11 Jun 2024 12:29:49 -0700 Subject: [PATCH 03/28] hmmm --- tests/notifications_utils/test_logging.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/notifications_utils/test_logging.py b/tests/notifications_utils/test_logging.py index 8f4e53923..493caabae 100644 --- a/tests/notifications_utils/test_logging.py +++ b/tests/notifications_utils/test_logging.py @@ -62,5 +62,5 @@ def test_pii_filter(): args=None, ) pii_filter = logging.PIIFilter() - x = pii_filter.filter(record) - assert pii_filter.filter(record).msg == "phone1: 1XXXXX55555, phone2: 1XXXXX55554, email1: XXXXXe@fake.gov, email2: XXXXX2.fake.gov" + clean_msg = "phone1: 1XXXXX55555, phone2: 1XXXXX55554, email1: XXXXXe@fake.gov, email2: XXXXX2.fake.gov" + assert pii_filter.filter(record).msg == clean_msg From 2a005076d046a1e6b683490cfc26ef788ca10673 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Wed, 12 Jun 2024 10:17:23 -0700 Subject: [PATCH 04/28] fix bug where only exception is passed to logger --- notifications_utils/logging.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/notifications_utils/logging.py b/notifications_utils/logging.py index f224533af..c1be8d400 100644 --- a/notifications_utils/logging.py +++ b/notifications_utils/logging.py @@ -123,7 +123,12 @@ class PIIFilter(logging.Filter): # and email addresses, masking them. Ultimately this will probably get # refactored into a 'SafeLogger' subclass or something, but let's start here # with phones. + + # Sometimes just an exception object is passed in for the message, skip those. + if not isinstance(msg, str): + return msg phones = re.findall("(?:\\+ *)?\\d[\\d\\- ]{7,}\\d", msg) + phones = [phone.replace("-", "").replace(" ", "") for phone in phones] for phone in phones: msg = msg.replace(phone, f"1XXXXX{phone[-5:]}") From 1c697202f386718ffff7f5a4f4161b8246ae0d97 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 13 Jun 2024 10:27:04 -0700 Subject: [PATCH 05/28] improve email masking --- notifications_utils/logging.py | 4 +++- tests/notifications_utils/test_logging.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/notifications_utils/logging.py b/notifications_utils/logging.py index c1be8d400..3de43e5db 100644 --- a/notifications_utils/logging.py +++ b/notifications_utils/logging.py @@ -138,7 +138,9 @@ class PIIFilter(logging.Filter): ) # ['alice@google.com', 'bob@abc.com'] for email in emails: # do something with each found email string - msg = msg.replace(email, f"XXXXX{email[-10:]}") + email_parts = email.split("@") + masked_email = f"{email_parts[0][0:3]}XXX@{email_parts[1][0:7]}XXX" + msg = msg.replace(email, masked_email) return msg def filter(self, record): diff --git a/tests/notifications_utils/test_logging.py b/tests/notifications_utils/test_logging.py index 493caabae..07b60bec0 100644 --- a/tests/notifications_utils/test_logging.py +++ b/tests/notifications_utils/test_logging.py @@ -62,5 +62,5 @@ def test_pii_filter(): args=None, ) pii_filter = logging.PIIFilter() - clean_msg = "phone1: 1XXXXX55555, phone2: 1XXXXX55554, email1: XXXXXe@fake.gov, email2: XXXXX2.fake.gov" + clean_msg = "phone1: 1XXXXX55555, phone2: 1XXXXX55554, email1: fakXXX@fake.goXXX, email2: fakXXX@fake2.fXXX" assert pii_filter.filter(record).msg == clean_msg From decbd254b0e422fc48c834cf87fc9e1250624628 Mon Sep 17 00:00:00 2001 From: John Skiles Skinner Date: Mon, 17 Jun 2024 17:23:00 -0700 Subject: [PATCH 06/28] Upgrade terraform-cloudgov, add prevent_destroy --- terraform/bootstrap/main.tf | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/terraform/bootstrap/main.tf b/terraform/bootstrap/main.tf index 14ab9a81d..a518530f1 100644 --- a/terraform/bootstrap/main.tf +++ b/terraform/bootstrap/main.tf @@ -3,7 +3,7 @@ locals { } module "s3" { - source = "github.com/18f/terraform-cloudgov//s3?ref=v0.9.1" + source = "github.com/GSA-TTS/terraform-cloudgov//s3?ref=v1.0.0" cf_org_name = "gsa-tts-benefits-studio" cf_space_name = "notify-management" @@ -13,4 +13,8 @@ module "s3" { resource "cloudfoundry_service_key" "bucket_creds" { name = "${local.s3_service_name}-access" service_instance = module.s3.bucket_id + + lifecycle { + prevent_destroy = true + } } From be000ba9596cb979596d7135fcea8da91f6b4687 Mon Sep 17 00:00:00 2001 From: John Skiles Skinner Date: Tue, 18 Jun 2024 16:42:12 -0700 Subject: [PATCH 07/28] Upgrade terraform-cloudgov, remove recursive_delete, add prevent_destroy --- terraform/staging/main.tf | 42 ++++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/terraform/staging/main.tf b/terraform/staging/main.tf index c46e0d3fa..c73273785 100644 --- a/terraform/staging/main.tf +++ b/terraform/staging/main.tf @@ -3,36 +3,40 @@ locals { cf_space_name = "notify-staging" env = "staging" app_name = "notify-api" - recursive_delete = true + recursive_delete = true # deprecated, still used in shared +} + +resource "null_resource" "prevent_destroy" { + + lifecycle { + prevent_destroy = true + } } module "database" { - source = "github.com/18f/terraform-cloudgov//database?ref=v0.7.1" + source = "github.com/GSA-TTS/terraform-cloudgov//database?ref=v1.0.0" - cf_org_name = local.cf_org_name - cf_space_name = local.cf_space_name - name = "${local.app_name}-rds-${local.env}" - recursive_delete = local.recursive_delete - rds_plan_name = "micro-psql" + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + name = "${local.app_name}-rds-${local.env}" + rds_plan_name = "micro-psql" } module "redis" { - source = "github.com/18f/terraform-cloudgov//redis?ref=v0.7.1" + source = "github.com/GSA-TTS/terraform-cloudgov//redis?ref=v1.0.0" - cf_org_name = local.cf_org_name - cf_space_name = local.cf_space_name - name = "${local.app_name}-redis-${local.env}" - recursive_delete = local.recursive_delete - redis_plan_name = "redis-dev" + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + name = "${local.app_name}-redis-${local.env}" + redis_plan_name = "redis-dev" } module "csv_upload_bucket" { - source = "github.com/18f/terraform-cloudgov//s3?ref=v0.7.1" + source = "github.com/GSA-TTS/terraform-cloudgov//s3?ref=v1.0.0" - cf_org_name = local.cf_org_name - cf_space_name = local.cf_space_name - recursive_delete = local.recursive_delete - name = "${local.app_name}-csv-upload-bucket-${local.env}" + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + name = "${local.app_name}-csv-upload-bucket-${local.env}" } module "egress-space" { @@ -53,7 +57,6 @@ module "ses_email" { cf_org_name = local.cf_org_name cf_space_name = local.cf_space_name name = "${local.app_name}-ses-${local.env}" - recursive_delete = local.recursive_delete aws_region = "us-west-2" mail_from_subdomain = "mail" email_receipt_error = "notify-support@gsa.gov" @@ -65,7 +68,6 @@ module "sns_sms" { cf_org_name = local.cf_org_name cf_space_name = local.cf_space_name name = "${local.app_name}-sns-${local.env}" - recursive_delete = local.recursive_delete aws_region = "us-west-2" monthly_spend_limit = 25 } From c57bdc2eba13aa99eb0a863b1608b2e046922b46 Mon Sep 17 00:00:00 2001 From: John Skiles Skinner Date: Tue, 18 Jun 2024 16:55:48 -0700 Subject: [PATCH 08/28] Make value of prevent_destroy match recursive_delete --- terraform/staging/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/staging/main.tf b/terraform/staging/main.tf index c73273785..39eb4b540 100644 --- a/terraform/staging/main.tf +++ b/terraform/staging/main.tf @@ -9,7 +9,7 @@ locals { resource "null_resource" "prevent_destroy" { lifecycle { - prevent_destroy = true + prevent_destroy = false # destrying staging is allowed } } From d84a199170f840e3dc64f6e7ae7230e7d3c214d0 Mon Sep 17 00:00:00 2001 From: John Skiles Skinner Date: Tue, 18 Jun 2024 16:56:44 -0700 Subject: [PATCH 09/28] Typo --- terraform/staging/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/staging/main.tf b/terraform/staging/main.tf index 39eb4b540..79d90d323 100644 --- a/terraform/staging/main.tf +++ b/terraform/staging/main.tf @@ -9,7 +9,7 @@ locals { resource "null_resource" "prevent_destroy" { lifecycle { - prevent_destroy = false # destrying staging is allowed + prevent_destroy = false # destroying staging is allowed } } From 5d592f1ab7f42b737d43961f2574f30f70c8c369 Mon Sep 17 00:00:00 2001 From: John Skiles Skinner Date: Tue, 18 Jun 2024 17:03:01 -0700 Subject: [PATCH 10/28] Upgrade terraform-cloudgov, remove recursive_delete, add prevent_destroy --- terraform/production/main.tf | 42 +++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/terraform/production/main.tf b/terraform/production/main.tf index 5a2c520b1..424eea904 100644 --- a/terraform/production/main.tf +++ b/terraform/production/main.tf @@ -3,36 +3,40 @@ locals { cf_space_name = "notify-production" env = "production" app_name = "notify-api" - recursive_delete = false + recursive_delete = false # deprecated, still used in shared +} + +resource "null_resource" "prevent_destroy" { + + lifecycle { + prevent_destroy = true + } } module "database" { - source = "github.com/18f/terraform-cloudgov//database?ref=v0.7.1" + source = "github.com/GSA-TTS/terraform-cloudgov//database?ref=v1.0.0" - cf_org_name = local.cf_org_name - cf_space_name = local.cf_space_name - name = "${local.app_name}-rds-${local.env}" - recursive_delete = local.recursive_delete - rds_plan_name = "small-psql-redundant" + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + name = "${local.app_name}-rds-${local.env}" + rds_plan_name = "small-psql-redundant" } module "redis" { - source = "github.com/18f/terraform-cloudgov//redis?ref=v0.7.1" + source = "github.com/GSA-TTS/terraform-cloudgov//redis?ref=v1.0.0" - cf_org_name = local.cf_org_name - cf_space_name = local.cf_space_name - name = "${local.app_name}-redis-${local.env}" - recursive_delete = local.recursive_delete - redis_plan_name = "redis-3node-large" + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + name = "${local.app_name}-redis-${local.env}" + redis_plan_name = "redis-3node-large" } module "csv_upload_bucket" { - source = "github.com/18f/terraform-cloudgov//s3?ref=v0.7.1" + source = "github.com/GSA-TTS/terraform-cloudgov//s3?ref=v1.0.0" - cf_org_name = local.cf_org_name - cf_space_name = local.cf_space_name - recursive_delete = local.recursive_delete - name = "${local.app_name}-csv-upload-bucket-${local.env}" + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + name = "${local.app_name}-csv-upload-bucket-${local.env}" } module "egress-space" { @@ -51,7 +55,6 @@ module "ses_email" { cf_org_name = local.cf_org_name cf_space_name = local.cf_space_name name = "${local.app_name}-ses-${local.env}" - recursive_delete = local.recursive_delete aws_region = "us-gov-west-1" email_domain = "notify.gov" mail_from_subdomain = "mail" @@ -64,7 +67,6 @@ module "sns_sms" { cf_org_name = local.cf_org_name cf_space_name = local.cf_space_name name = "${local.app_name}-sns-${local.env}" - recursive_delete = local.recursive_delete aws_region = "us-gov-west-1" monthly_spend_limit = 1000 } From 76cc718b32b5280d1edf88ad7ba620d7c977f87d Mon Sep 17 00:00:00 2001 From: John Skiles Skinner Date: Tue, 18 Jun 2024 17:26:08 -0700 Subject: [PATCH 11/28] Remove recursive_delete from SNS & SES shared modules --- terraform/demo/main.tf | 3 --- terraform/production/main.tf | 3 --- terraform/sandbox/main.tf | 3 --- terraform/shared/ses/main.tf | 1 - terraform/shared/ses/variables.tf | 6 ------ terraform/shared/sns/main.tf | 1 - terraform/shared/sns/variables.tf | 6 ------ terraform/staging/main.tf | 3 --- 8 files changed, 26 deletions(-) diff --git a/terraform/demo/main.tf b/terraform/demo/main.tf index e252ddf6d..c335916e1 100644 --- a/terraform/demo/main.tf +++ b/terraform/demo/main.tf @@ -3,7 +3,6 @@ locals { cf_space_name = "notify-demo" env = "demo" app_name = "notify-api" - recursive_delete = false } module "database" { @@ -66,7 +65,6 @@ module "ses_email" { cf_org_name = local.cf_org_name cf_space_name = local.cf_space_name name = "${local.app_name}-ses-${local.env}" - recursive_delete = local.recursive_delete aws_region = "us-west-2" email_domain = "notify.sandbox.10x.gsa.gov" email_receipt_error = "notify-support@gsa.gov" @@ -78,7 +76,6 @@ module "sns_sms" { cf_org_name = local.cf_org_name cf_space_name = local.cf_space_name name = "${local.app_name}-sns-${local.env}" - recursive_delete = local.recursive_delete aws_region = "us-east-1" monthly_spend_limit = 25 } diff --git a/terraform/production/main.tf b/terraform/production/main.tf index 5a2c520b1..b0bd2f319 100644 --- a/terraform/production/main.tf +++ b/terraform/production/main.tf @@ -3,7 +3,6 @@ locals { cf_space_name = "notify-production" env = "production" app_name = "notify-api" - recursive_delete = false } module "database" { @@ -51,7 +50,6 @@ module "ses_email" { cf_org_name = local.cf_org_name cf_space_name = local.cf_space_name name = "${local.app_name}-ses-${local.env}" - recursive_delete = local.recursive_delete aws_region = "us-gov-west-1" email_domain = "notify.gov" mail_from_subdomain = "mail" @@ -64,7 +62,6 @@ module "sns_sms" { cf_org_name = local.cf_org_name cf_space_name = local.cf_space_name name = "${local.app_name}-sns-${local.env}" - recursive_delete = local.recursive_delete aws_region = "us-gov-west-1" monthly_spend_limit = 1000 } diff --git a/terraform/sandbox/main.tf b/terraform/sandbox/main.tf index b0b6381bd..29879fe83 100644 --- a/terraform/sandbox/main.tf +++ b/terraform/sandbox/main.tf @@ -3,7 +3,6 @@ locals { cf_space_name = "notify-sandbox" env = "sandbox" app_name = "notify-api" - recursive_delete = true # deprecated, still used in shared modules } resource "null_resource" "prevent_destroy" { @@ -71,7 +70,6 @@ module "ses_email" { cf_org_name = local.cf_org_name cf_space_name = local.cf_space_name name = "${local.app_name}-ses-${local.env}" - recursive_delete = local.recursive_delete aws_region = "us-west-2" email_receipt_error = "notify-support@gsa.gov" } @@ -82,7 +80,6 @@ module "sns_sms" { cf_org_name = local.cf_org_name cf_space_name = local.cf_space_name name = "${local.app_name}-sns-${local.env}" - recursive_delete = local.recursive_delete aws_region = "us-east-2" monthly_spend_limit = 1 } diff --git a/terraform/shared/ses/main.tf b/terraform/shared/ses/main.tf index a29a8ce10..016607efa 100644 --- a/terraform/shared/ses/main.tf +++ b/terraform/shared/ses/main.tf @@ -19,7 +19,6 @@ resource "cloudfoundry_service_instance" "ses" { name = var.name space = data.cloudfoundry_space.space.id service_plan = data.cloudfoundry_service.ses.service_plans["base"] - recursive_delete = var.recursive_delete json_params = jsonencode({ region = var.aws_region domain = var.email_domain diff --git a/terraform/shared/ses/variables.tf b/terraform/shared/ses/variables.tf index 74e852cf6..a92261656 100644 --- a/terraform/shared/ses/variables.tf +++ b/terraform/shared/ses/variables.tf @@ -13,12 +13,6 @@ variable "name" { description = "name of the service instance" } -variable "recursive_delete" { - type = bool - description = "when true, deletes service bindings attached to the resource (not recommended for production)" - default = false -} - variable "aws_region" { type = string description = "AWS region the SES instance is in" diff --git a/terraform/shared/sns/main.tf b/terraform/shared/sns/main.tf index a23c4e872..c7c8df6f9 100644 --- a/terraform/shared/sns/main.tf +++ b/terraform/shared/sns/main.tf @@ -19,7 +19,6 @@ resource "cloudfoundry_service_instance" "sns" { name = var.name space = data.cloudfoundry_space.space.id service_plan = data.cloudfoundry_service.sns.service_plans["base"] - recursive_delete = var.recursive_delete json_params = jsonencode({ region = var.aws_region monthly_spend_limit = var.monthly_spend_limit diff --git a/terraform/shared/sns/variables.tf b/terraform/shared/sns/variables.tf index 611050337..acf7c5010 100644 --- a/terraform/shared/sns/variables.tf +++ b/terraform/shared/sns/variables.tf @@ -13,12 +13,6 @@ variable "name" { description = "name of the service instance" } -variable "recursive_delete" { - type = bool - description = "when true, deletes service bindings attached to the resource (not recommended for production)" - default = false -} - variable "aws_region" { type = string description = "AWS region the SNS settings are set in" diff --git a/terraform/staging/main.tf b/terraform/staging/main.tf index c46e0d3fa..6dc81928e 100644 --- a/terraform/staging/main.tf +++ b/terraform/staging/main.tf @@ -3,7 +3,6 @@ locals { cf_space_name = "notify-staging" env = "staging" app_name = "notify-api" - recursive_delete = true } module "database" { @@ -53,7 +52,6 @@ module "ses_email" { cf_org_name = local.cf_org_name cf_space_name = local.cf_space_name name = "${local.app_name}-ses-${local.env}" - recursive_delete = local.recursive_delete aws_region = "us-west-2" mail_from_subdomain = "mail" email_receipt_error = "notify-support@gsa.gov" @@ -65,7 +63,6 @@ module "sns_sms" { cf_org_name = local.cf_org_name cf_space_name = local.cf_space_name name = "${local.app_name}-sns-${local.env}" - recursive_delete = local.recursive_delete aws_region = "us-west-2" monthly_spend_limit = 25 } From fde4c47e31379c0b7b9c1dc3bc0d812c5a725165 Mon Sep 17 00:00:00 2001 From: John Skiles Skinner Date: Tue, 18 Jun 2024 17:43:38 -0700 Subject: [PATCH 12/28] Terraform format the SNS and SES modules --- terraform/shared/ses/main.tf | 6 +++--- terraform/shared/sns/main.tf | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/terraform/shared/ses/main.tf b/terraform/shared/ses/main.tf index 016607efa..4c1bb54b9 100644 --- a/terraform/shared/ses/main.tf +++ b/terraform/shared/ses/main.tf @@ -16,9 +16,9 @@ data "cloudfoundry_service" "ses" { } resource "cloudfoundry_service_instance" "ses" { - name = var.name - space = data.cloudfoundry_space.space.id - service_plan = data.cloudfoundry_service.ses.service_plans["base"] + name = var.name + space = data.cloudfoundry_space.space.id + service_plan = data.cloudfoundry_service.ses.service_plans["base"] json_params = jsonencode({ region = var.aws_region domain = var.email_domain diff --git a/terraform/shared/sns/main.tf b/terraform/shared/sns/main.tf index c7c8df6f9..aa0079f92 100644 --- a/terraform/shared/sns/main.tf +++ b/terraform/shared/sns/main.tf @@ -16,9 +16,9 @@ data "cloudfoundry_service" "sns" { } resource "cloudfoundry_service_instance" "sns" { - name = var.name - space = data.cloudfoundry_space.space.id - service_plan = data.cloudfoundry_service.sns.service_plans["base"] + name = var.name + space = data.cloudfoundry_space.space.id + service_plan = data.cloudfoundry_service.sns.service_plans["base"] json_params = jsonencode({ region = var.aws_region monthly_spend_limit = var.monthly_spend_limit From 6449c13312f9bb847ea6e35798c3e8c2eb34db04 Mon Sep 17 00:00:00 2001 From: John Skiles Skinner Date: Tue, 18 Jun 2024 17:46:36 -0700 Subject: [PATCH 13/28] Running fmt to create a tf plan output file to make PR pass --- terraform/demo/main.tf | 8 ++++---- terraform/production/main.tf | 8 ++++---- terraform/sandbox/main.tf | 32 ++++++++++++++++---------------- terraform/staging/main.tf | 8 ++++---- 4 files changed, 28 insertions(+), 28 deletions(-) diff --git a/terraform/demo/main.tf b/terraform/demo/main.tf index c335916e1..399861abd 100644 --- a/terraform/demo/main.tf +++ b/terraform/demo/main.tf @@ -1,8 +1,8 @@ locals { - cf_org_name = "gsa-tts-benefits-studio" - cf_space_name = "notify-demo" - env = "demo" - app_name = "notify-api" + cf_org_name = "gsa-tts-benefits-studio" + cf_space_name = "notify-demo" + env = "demo" + app_name = "notify-api" } module "database" { diff --git a/terraform/production/main.tf b/terraform/production/main.tf index b0bd2f319..9a159e663 100644 --- a/terraform/production/main.tf +++ b/terraform/production/main.tf @@ -1,8 +1,8 @@ locals { - cf_org_name = "gsa-tts-benefits-studio" - cf_space_name = "notify-production" - env = "production" - app_name = "notify-api" + cf_org_name = "gsa-tts-benefits-studio" + cf_space_name = "notify-production" + env = "production" + app_name = "notify-api" } module "database" { diff --git a/terraform/sandbox/main.tf b/terraform/sandbox/main.tf index 29879fe83..319b273ba 100644 --- a/terraform/sandbox/main.tf +++ b/terraform/sandbox/main.tf @@ -1,8 +1,8 @@ locals { - cf_org_name = "gsa-tts-benefits-studio" - cf_space_name = "notify-sandbox" - env = "sandbox" - app_name = "notify-api" + cf_org_name = "gsa-tts-benefits-studio" + cf_space_name = "notify-sandbox" + env = "sandbox" + app_name = "notify-api" } resource "null_resource" "prevent_destroy" { @@ -15,19 +15,19 @@ resource "null_resource" "prevent_destroy" { module "database" { source = "github.com/GSA-TTS/terraform-cloudgov//database?ref=v1.0.0" - cf_org_name = local.cf_org_name - cf_space_name = local.cf_space_name - name = "${local.app_name}-rds-${local.env}" - rds_plan_name = "micro-psql" + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + name = "${local.app_name}-rds-${local.env}" + rds_plan_name = "micro-psql" } module "redis" { # default v6.2; delete after v7.0 resource is bound source = "github.com/GSA-TTS/terraform-cloudgov//redis?ref=v1.0.0" - cf_org_name = local.cf_org_name - cf_space_name = local.cf_space_name - name = "${local.app_name}-redis-${local.env}" - redis_plan_name = "redis-dev" + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + name = "${local.app_name}-redis-${local.env}" + redis_plan_name = "redis-dev" } module "redis-v70" { @@ -37,7 +37,7 @@ module "redis-v70" { cf_space_name = local.cf_space_name name = "${local.app_name}-redis-v70-${local.env}" redis_plan_name = "redis-dev" - json_params = jsonencode( + json_params = jsonencode( { "engineVersion" : "7.0", } @@ -47,9 +47,9 @@ module "redis-v70" { module "csv_upload_bucket" { source = "github.com/GSA-TTS/terraform-cloudgov//s3?ref=v1.0.0" - cf_org_name = local.cf_org_name - cf_space_name = local.cf_space_name - name = "${local.app_name}-csv-upload-bucket-${local.env}" + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + name = "${local.app_name}-csv-upload-bucket-${local.env}" } module "egress-space" { diff --git a/terraform/staging/main.tf b/terraform/staging/main.tf index 6dc81928e..ec0e00ec3 100644 --- a/terraform/staging/main.tf +++ b/terraform/staging/main.tf @@ -1,8 +1,8 @@ locals { - cf_org_name = "gsa-tts-benefits-studio" - cf_space_name = "notify-staging" - env = "staging" - app_name = "notify-api" + cf_org_name = "gsa-tts-benefits-studio" + cf_space_name = "notify-staging" + env = "staging" + app_name = "notify-api" } module "database" { From 6ac5be1d87bab508bff44b85505cbd17ddbe7d71 Mon Sep 17 00:00:00 2001 From: John Skiles Skinner Date: Thu, 20 Jun 2024 09:37:19 -0700 Subject: [PATCH 14/28] Add Redis v7.0 module, terraform-cloudgov v1.0.0 to Staging space --- terraform/staging/main.tf | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/terraform/staging/main.tf b/terraform/staging/main.tf index c46e0d3fa..c0f4b1db3 100644 --- a/terraform/staging/main.tf +++ b/terraform/staging/main.tf @@ -16,7 +16,7 @@ module "database" { rds_plan_name = "micro-psql" } -module "redis" { +module "redis" { # default v6.2; delete after v7.0 resource is bound source = "github.com/18f/terraform-cloudgov//redis?ref=v0.7.1" cf_org_name = local.cf_org_name @@ -26,6 +26,20 @@ module "redis" { redis_plan_name = "redis-dev" } +module "redis-v70" { + source = "github.com/GSA-TTS/terraform-cloudgov//redis?ref=v1.0.0" + + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + name = "${local.app_name}-redis-v70-${local.env}" + redis_plan_name = "redis-dev" + json_params = jsonencode( + { + "engineVersion" : "7.0", + } + ) +} + module "csv_upload_bucket" { source = "github.com/18f/terraform-cloudgov//s3?ref=v0.7.1" From 8e23a6c8f28499f88a2c21f8ec9dd399cb52648a Mon Sep 17 00:00:00 2001 From: John Skiles Skinner Date: Thu, 20 Jun 2024 09:40:55 -0700 Subject: [PATCH 15/28] Same in Production space --- terraform/production/main.tf | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/terraform/production/main.tf b/terraform/production/main.tf index 5a2c520b1..f81a9083e 100644 --- a/terraform/production/main.tf +++ b/terraform/production/main.tf @@ -16,7 +16,7 @@ module "database" { rds_plan_name = "small-psql-redundant" } -module "redis" { +module "redis" { # default v6.2; delete after v7.0 resource is bound source = "github.com/18f/terraform-cloudgov//redis?ref=v0.7.1" cf_org_name = local.cf_org_name @@ -26,6 +26,20 @@ module "redis" { redis_plan_name = "redis-3node-large" } +module "redis-v70" { + source = "github.com/GSA-TTS/terraform-cloudgov//redis?ref=v1.0.0" + + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + name = "${local.app_name}-redis-v70-${local.env}" + redis_plan_name = "redis-dev" + json_params = jsonencode( + { + "engineVersion" : "7.0", + } + ) +} + module "csv_upload_bucket" { source = "github.com/18f/terraform-cloudgov//s3?ref=v0.7.1" From 360d4f2a9a2160df1971e1406e5f338d10778fe2 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 20 Jun 2024 11:55:01 -0700 Subject: [PATCH 16/28] fix raw_set and raw_get --- .../test_service_invite_rest.py | 8 ++++---- tests/app/user/test_rest_verify.py | 20 +++++++++---------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/tests/app/service_invite/test_service_invite_rest.py b/tests/app/service_invite/test_service_invite_rest.py index 0f60dc3e3..327729776 100644 --- a/tests/app/service_invite/test_service_invite_rest.py +++ b/tests/app/service_invite/test_service_invite_rest.py @@ -96,8 +96,8 @@ def test_create_invited_user_without_auth_type( admin_request, sample_service, mocker, invitation_email_template ): - mocker.patch("app.service_invite.rest.redis_store.raw_set") - mocker.patch("app.service_invite.rest.redis_store.raw_get") + mocker.patch("app.service_invite.rest.redis_store.set") + mocker.patch("app.service_invite.rest.redis_store.get") mocker.patch("app.celery.provider_tasks.deliver_email.apply_async") email_address = "invited_user@service.gov.uk" invite_from = sample_service.users[0] @@ -220,8 +220,8 @@ def test_resend_expired_invite( mocker, ): - mocker.patch("app.service_invite.rest.redis_store.raw_set") - mocker.patch("app.service_invite.rest.redis_store.raw_get") + mocker.patch("app.service_invite.rest.redis_store.set") + mocker.patch("app.service_invite.rest.redis_store.get") url = f"/service/{sample_expired_user.service_id}/invite/{sample_expired_user.id}/resend" mock_send = mocker.patch("app.service_invite.rest.send_notification_to_queue") mock_persist = mocker.patch("app.service_invite.rest.persist_notification") diff --git a/tests/app/user/test_rest_verify.py b/tests/app/user/test_rest_verify.py index 81d813039..ff74f6b57 100644 --- a/tests/app/user/test_rest_verify.py +++ b/tests/app/user/test_rest_verify.py @@ -201,10 +201,10 @@ def test_send_user_sms_code(client, sample_user, sms_code_template, mocker): """ notify_service = dao_fetch_service_by_id(current_app.config["NOTIFY_SERVICE_ID"]) - mock_redis_get = mocker.patch("app.user.rest.redis_store.raw_get") + mock_redis_get = mocker.patch("app.user.rest.redis_store.get") mock_redis_get.return_value = "foo" - mocker.patch("app.user.rest.redis_store.raw_set") + mocker.patch("app.user.rest.redis_store.set") auth_header = create_admin_authorization_header() mocked = mocker.patch("app.user.rest.create_secret_code", return_value="11111") mocker.patch("app.celery.provider_tasks.deliver_sms.apply_async") @@ -242,10 +242,10 @@ def test_send_user_code_for_sms_with_optional_to_field( Tests POST endpoint /user//sms-code with optional to field """ - mock_redis_get = mocker.patch("app.user.rest.redis_store.raw_get") + mock_redis_get = mocker.patch("app.user.rest.redis_store.get") mock_redis_get.return_value = "foo" - mocker.patch("app.user.rest.redis_store.raw_set") + mocker.patch("app.user.rest.redis_store.set") to_number = "+14254147755" mocked = mocker.patch("app.user.rest.create_secret_code", return_value="11111") mocker.patch("app.celery.provider_tasks.deliver_sms.apply_async") @@ -469,10 +469,10 @@ def test_send_user_email_code( deliver_email = mocker.patch("app.celery.provider_tasks.deliver_email.apply_async") sample_user.auth_type = auth_type - mock_redis_get = mocker.patch("app.user.rest.redis_store.raw_get") + mock_redis_get = mocker.patch("app.user.rest.redis_store.get") mock_redis_get.return_value = "foo" - mocker.patch("app.user.rest.redis_store.raw_set") + mocker.patch("app.user.rest.redis_store.set") admin_request.post( "user.send_user_2fa_code", @@ -497,10 +497,10 @@ def test_send_user_email_code_with_urlencoded_next_param( ): mocker.patch("app.celery.provider_tasks.deliver_email.apply_async") - mock_redis_get = mocker.patch("app.celery.scheduled_tasks.redis_store.raw_get") + mock_redis_get = mocker.patch("app.celery.scheduled_tasks.redis_store.get") mock_redis_get.return_value = "foo" - mocker.patch("app.celery.scheduled_tasks.redis_store.raw_set") + mocker.patch("app.celery.scheduled_tasks.redis_store.set") data = {"to": None, "next": "/services"} admin_request.post( @@ -581,10 +581,10 @@ def test_user_verify_email_code_fails_if_code_already_used( def test_send_user_2fa_code_sends_from_number_for_international_numbers( client, sample_user, mocker, sms_code_template ): - mock_redis_get = mocker.patch("app.user.rest.redis_store.raw_get") + mock_redis_get = mocker.patch("app.user.rest.redis_store.get") mock_redis_get.return_value = "foo" - mocker.patch("app.user.rest.redis_store.raw_set") + mocker.patch("app.user.rest.redis_store.set") sample_user.mobile_number = "+601117224412" auth_header = create_admin_authorization_header() From 2e4fd3b3ac6f844563b48ce229e0dbdcdc4e265a Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 20 Jun 2024 13:30:43 -0700 Subject: [PATCH 17/28] fix tests --- app/delivery/send_to_providers.py | 2 +- app/user/rest.py | 2 +- .../clients/redis/redis_client.py | 16 ++---------- .../test_service_invite_rest.py | 4 +-- .../clients/redis/test_redis_client.py | 26 +------------------ 5 files changed, 7 insertions(+), 43 deletions(-) diff --git a/app/delivery/send_to_providers.py b/app/delivery/send_to_providers.py index a347b89ad..b6872620b 100644 --- a/app/delivery/send_to_providers.py +++ b/app/delivery/send_to_providers.py @@ -89,7 +89,7 @@ def send_sms_to_provider(notification): except Exception: # It is our 2facode, maybe key = f"2facode-{notification.id}".replace(" ", "") - recipient = redis_store.raw_get(key) + recipient = redis_store.get(key) if recipient: recipient = recipient.decode("utf-8") diff --git a/app/user/rest.py b/app/user/rest.py index 760334841..049549f2c 100644 --- a/app/user/rest.py +++ b/app/user/rest.py @@ -361,7 +361,7 @@ def create_2fa_code( saved_notification.personalisation = personalisation key = f"2facode-{saved_notification.id}".replace(" ", "") recipient = str(recipient) - redis_store.raw_set(key, recipient, ex=60 * 60) + redis_store.set(key, recipient, ex=60 * 60) # Assume that we never want to observe the Notify service's research mode # setting for this notification - we still need to be able to log into the diff --git a/notifications_utils/clients/redis/redis_client.py b/notifications_utils/clients/redis/redis_client.py index 6e81d85bf..1723dd2c1 100644 --- a/notifications_utils/clients/redis/redis_client.py +++ b/notifications_utils/clients/redis/redis_client.py @@ -133,19 +133,13 @@ class RedisClient: else: return False - def raw_set(self, key, value, ex=None, px=None, nx=False, xx=False): - self.redis_store.set(key, value, ex, px, nx, xx) - def set( self, key, value, ex=None, px=None, nx=False, xx=False, raise_exception=False ): key = prepare_value(key) value = prepare_value(value) if self.active: - try: - self.redis_store.set(key, value, ex, px, nx, xx) - except Exception as e: - self.__handle_exception(e, raise_exception, "set", key) + self.redis_store.set(key, value, ex, px, nx, xx) def incr(self, key, raise_exception=False): key = prepare_value(key) @@ -155,16 +149,10 @@ class RedisClient: except Exception as e: self.__handle_exception(e, raise_exception, "incr", key) - def raw_get(self, key): - return self.redis_store.get(key) - def get(self, key, raise_exception=False): key = prepare_value(key) if self.active: - try: - return self.redis_store.get(key) - except Exception as e: - self.__handle_exception(e, raise_exception, "get", key) + return self.redis_store.get(key) return None diff --git a/tests/app/service_invite/test_service_invite_rest.py b/tests/app/service_invite/test_service_invite_rest.py index 327729776..07d0b4c23 100644 --- a/tests/app/service_invite/test_service_invite_rest.py +++ b/tests/app/service_invite/test_service_invite_rest.py @@ -31,8 +31,8 @@ def test_create_invited_user( extra_args, expected_start_of_invite_url, ): - mocker.patch("app.service_invite.rest.redis_store.raw_set") - mocker.patch("app.service_invite.rest.redis_store.raw_get") + mocker.patch("app.service_invite.rest.redis_store.set") + mocker.patch("app.service_invite.rest.redis_store.get") mocked = mocker.patch("app.celery.provider_tasks.deliver_email.apply_async") email_address = "invited_user@service.gov.uk" diff --git a/tests/notifications_utils/clients/redis/test_redis_client.py b/tests/notifications_utils/clients/redis/test_redis_client.py index c9eb63240..52b227128 100644 --- a/tests/notifications_utils/clients/redis/test_redis_client.py +++ b/tests/notifications_utils/clients/redis/test_redis_client.py @@ -1,5 +1,5 @@ import uuid -from unittest.mock import Mock, call +from unittest.mock import Mock import pytest from freezegun import freeze_time @@ -59,30 +59,6 @@ def failing_redis_client(mocked_redis_client, delete_mock): return mocked_redis_client -def test_should_not_raise_exception_if_raise_set_to_false( - app, caplog, failing_redis_client, mocker -): - mock_logger = mocker.patch("flask.Flask.logger") - - assert failing_redis_client.get("get_key") is None - assert failing_redis_client.set("set_key", "set_value") is None - assert failing_redis_client.incr("incr_key") is None - assert failing_redis_client.exceeded_rate_limit("rate_limit_key", 100, 100) is False - assert failing_redis_client.delete("delete_key") is None - assert failing_redis_client.delete("a", "b", "c") is None - assert failing_redis_client.delete_by_pattern("pattern") == 0 - - assert mock_logger.mock_calls == [ - call.exception("Redis error performing get on get_key"), - call.exception("Redis error performing set on set_key"), - call.exception("Redis error performing incr on incr_key"), - call.exception("Redis error performing rate-limit-pipeline on rate_limit_key"), - call.exception("Redis error performing delete on delete_key"), - call.exception("Redis error performing delete on a, b, c"), - call.exception("Redis error performing delete-by-pattern on pattern"), - ] - - def test_should_raise_exception_if_raise_set_to_true( app, failing_redis_client, From 750d43df1438517e179458096ada5c19fba41c0d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 21 Jun 2024 21:39:13 +0000 Subject: [PATCH 18/28] Bump sqlalchemy from 2.0.30 to 2.0.31 Bumps [sqlalchemy](https://github.com/sqlalchemy/sqlalchemy) from 2.0.30 to 2.0.31. - [Release notes](https://github.com/sqlalchemy/sqlalchemy/releases) - [Changelog](https://github.com/sqlalchemy/sqlalchemy/blob/main/CHANGES.rst) - [Commits](https://github.com/sqlalchemy/sqlalchemy/commits) --- updated-dependencies: - dependency-name: sqlalchemy dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- poetry.lock | 112 +++++++++++++++++++++++++------------------------ pyproject.toml | 2 +- 2 files changed, 59 insertions(+), 55 deletions(-) diff --git a/poetry.lock b/poetry.lock index 29222296c..47991d267 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. [[package]] name = "aiohttp" @@ -2098,9 +2098,13 @@ files = [ {file = "lxml-5.2.2-cp36-cp36m-win_amd64.whl", hash = "sha256:edcfa83e03370032a489430215c1e7783128808fd3e2e0a3225deee278585196"}, {file = "lxml-5.2.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:28bf95177400066596cdbcfc933312493799382879da504633d16cf60bba735b"}, {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3a745cc98d504d5bd2c19b10c79c61c7c3df9222629f1b6210c0368177589fb8"}, + {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1b590b39ef90c6b22ec0be925b211298e810b4856909c8ca60d27ffbca6c12e6"}, {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b336b0416828022bfd5a2e3083e7f5ba54b96242159f83c7e3eebaec752f1716"}, + {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_28_aarch64.whl", hash = "sha256:c2faf60c583af0d135e853c86ac2735ce178f0e338a3c7f9ae8f622fd2eb788c"}, {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:4bc6cb140a7a0ad1f7bc37e018d0ed690b7b6520ade518285dc3171f7a117905"}, + {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7ff762670cada8e05b32bf1e4dc50b140790909caa8303cfddc4d702b71ea184"}, {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:57f0a0bbc9868e10ebe874e9f129d2917750adf008fe7b9c1598c0fbbfdde6a6"}, + {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:a6d2092797b388342c1bc932077ad232f914351932353e2e8706851c870bca1f"}, {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:60499fe961b21264e17a471ec296dcbf4365fbea611bf9e303ab69db7159ce61"}, {file = "lxml-5.2.2-cp37-cp37m-win32.whl", hash = "sha256:d9b342c76003c6b9336a80efcc766748a333573abf9350f4094ee46b006ec18f"}, {file = "lxml-5.2.2-cp37-cp37m-win_amd64.whl", hash = "sha256:b16db2770517b8799c79aa80f4053cd6f8b716f21f8aca962725a9565ce3ee40"}, @@ -2489,7 +2493,6 @@ files = [ {file = "msgpack-1.0.8-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5fbb160554e319f7b22ecf530a80a3ff496d38e8e07ae763b9e82fadfe96f273"}, {file = "msgpack-1.0.8-cp39-cp39-win32.whl", hash = "sha256:f9af38a89b6a5c04b7d18c492c8ccf2aee7048aff1ce8437c4683bb5a1df893d"}, {file = "msgpack-1.0.8-cp39-cp39-win_amd64.whl", hash = "sha256:ed59dd52075f8fc91da6053b12e8c89e37aa043f8986efd89e61fae69dc1b011"}, - {file = "msgpack-1.0.8-py3-none-any.whl", hash = "sha256:24f727df1e20b9876fa6e95f840a2a2651e34c0ad147676356f4bf5fbb0206ca"}, {file = "msgpack-1.0.8.tar.gz", hash = "sha256:95c02b0e27e706e48d0e5426d1710ca78e0f0628d6e89d5b5a5b91a5f12274f3"}, ] @@ -3472,6 +3475,7 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, @@ -4143,64 +4147,64 @@ files = [ [[package]] name = "sqlalchemy" -version = "2.0.30" +version = "2.0.31" description = "Database Abstraction Library" optional = false python-versions = ">=3.7" files = [ - {file = "SQLAlchemy-2.0.30-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3b48154678e76445c7ded1896715ce05319f74b1e73cf82d4f8b59b46e9c0ddc"}, - {file = "SQLAlchemy-2.0.30-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2753743c2afd061bb95a61a51bbb6a1a11ac1c44292fad898f10c9839a7f75b2"}, - {file = "SQLAlchemy-2.0.30-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a7bfc726d167f425d4c16269a9a10fe8630ff6d14b683d588044dcef2d0f6be7"}, - {file = "SQLAlchemy-2.0.30-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c4f61ada6979223013d9ab83a3ed003ded6959eae37d0d685db2c147e9143797"}, - {file = "SQLAlchemy-2.0.30-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:3a365eda439b7a00732638f11072907c1bc8e351c7665e7e5da91b169af794af"}, - {file = "SQLAlchemy-2.0.30-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bba002a9447b291548e8d66fd8c96a6a7ed4f2def0bb155f4f0a1309fd2735d5"}, - {file = "SQLAlchemy-2.0.30-cp310-cp310-win32.whl", hash = "sha256:0138c5c16be3600923fa2169532205d18891b28afa817cb49b50e08f62198bb8"}, - {file = "SQLAlchemy-2.0.30-cp310-cp310-win_amd64.whl", hash = "sha256:99650e9f4cf3ad0d409fed3eec4f071fadd032e9a5edc7270cd646a26446feeb"}, - {file = "SQLAlchemy-2.0.30-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:955991a09f0992c68a499791a753523f50f71a6885531568404fa0f231832aa0"}, - {file = "SQLAlchemy-2.0.30-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f69e4c756ee2686767eb80f94c0125c8b0a0b87ede03eacc5c8ae3b54b99dc46"}, - {file = "SQLAlchemy-2.0.30-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69c9db1ce00e59e8dd09d7bae852a9add716efdc070a3e2068377e6ff0d6fdaa"}, - {file = "SQLAlchemy-2.0.30-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a1429a4b0f709f19ff3b0cf13675b2b9bfa8a7e79990003207a011c0db880a13"}, - {file = "SQLAlchemy-2.0.30-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:efedba7e13aa9a6c8407c48facfdfa108a5a4128e35f4c68f20c3407e4376aa9"}, - {file = "SQLAlchemy-2.0.30-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:16863e2b132b761891d6c49f0a0f70030e0bcac4fd208117f6b7e053e68668d0"}, - {file = "SQLAlchemy-2.0.30-cp311-cp311-win32.whl", hash = "sha256:2ecabd9ccaa6e914e3dbb2aa46b76dede7eadc8cbf1b8083c94d936bcd5ffb49"}, - {file = "SQLAlchemy-2.0.30-cp311-cp311-win_amd64.whl", hash = "sha256:0b3f4c438e37d22b83e640f825ef0f37b95db9aa2d68203f2c9549375d0b2260"}, - {file = "SQLAlchemy-2.0.30-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5a79d65395ac5e6b0c2890935bad892eabb911c4aa8e8015067ddb37eea3d56c"}, - {file = "SQLAlchemy-2.0.30-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9a5baf9267b752390252889f0c802ea13b52dfee5e369527da229189b8bd592e"}, - {file = "SQLAlchemy-2.0.30-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3cb5a646930c5123f8461f6468901573f334c2c63c795b9af350063a736d0134"}, - {file = "SQLAlchemy-2.0.30-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:296230899df0b77dec4eb799bcea6fbe39a43707ce7bb166519c97b583cfcab3"}, - {file = "SQLAlchemy-2.0.30-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c62d401223f468eb4da32627bffc0c78ed516b03bb8a34a58be54d618b74d472"}, - {file = "SQLAlchemy-2.0.30-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3b69e934f0f2b677ec111b4d83f92dc1a3210a779f69bf905273192cf4ed433e"}, - {file = "SQLAlchemy-2.0.30-cp312-cp312-win32.whl", hash = "sha256:77d2edb1f54aff37e3318f611637171e8ec71472f1fdc7348b41dcb226f93d90"}, - {file = "SQLAlchemy-2.0.30-cp312-cp312-win_amd64.whl", hash = "sha256:b6c7ec2b1f4969fc19b65b7059ed00497e25f54069407a8701091beb69e591a5"}, - {file = "SQLAlchemy-2.0.30-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5a8e3b0a7e09e94be7510d1661339d6b52daf202ed2f5b1f9f48ea34ee6f2d57"}, - {file = "SQLAlchemy-2.0.30-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b60203c63e8f984df92035610c5fb76d941254cf5d19751faab7d33b21e5ddc0"}, - {file = "SQLAlchemy-2.0.30-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f1dc3eabd8c0232ee8387fbe03e0a62220a6f089e278b1f0aaf5e2d6210741ad"}, - {file = "SQLAlchemy-2.0.30-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:40ad017c672c00b9b663fcfcd5f0864a0a97828e2ee7ab0c140dc84058d194cf"}, - {file = "SQLAlchemy-2.0.30-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:e42203d8d20dc704604862977b1470a122e4892791fe3ed165f041e4bf447a1b"}, - {file = "SQLAlchemy-2.0.30-cp37-cp37m-win32.whl", hash = "sha256:2a4f4da89c74435f2bc61878cd08f3646b699e7d2eba97144030d1be44e27584"}, - {file = "SQLAlchemy-2.0.30-cp37-cp37m-win_amd64.whl", hash = "sha256:b6bf767d14b77f6a18b6982cbbf29d71bede087edae495d11ab358280f304d8e"}, - {file = "SQLAlchemy-2.0.30-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:bc0c53579650a891f9b83fa3cecd4e00218e071d0ba00c4890f5be0c34887ed3"}, - {file = "SQLAlchemy-2.0.30-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:311710f9a2ee235f1403537b10c7687214bb1f2b9ebb52702c5aa4a77f0b3af7"}, - {file = "SQLAlchemy-2.0.30-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:408f8b0e2c04677e9c93f40eef3ab22f550fecb3011b187f66a096395ff3d9fd"}, - {file = "SQLAlchemy-2.0.30-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37a4b4fb0dd4d2669070fb05b8b8824afd0af57587393015baee1cf9890242d9"}, - {file = "SQLAlchemy-2.0.30-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:a943d297126c9230719c27fcbbeab57ecd5d15b0bd6bfd26e91bfcfe64220621"}, - {file = "SQLAlchemy-2.0.30-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:0a089e218654e740a41388893e090d2e2c22c29028c9d1353feb38638820bbeb"}, - {file = "SQLAlchemy-2.0.30-cp38-cp38-win32.whl", hash = "sha256:fa561138a64f949f3e889eb9ab8c58e1504ab351d6cf55259dc4c248eaa19da6"}, - {file = "SQLAlchemy-2.0.30-cp38-cp38-win_amd64.whl", hash = "sha256:7d74336c65705b986d12a7e337ba27ab2b9d819993851b140efdf029248e818e"}, - {file = "SQLAlchemy-2.0.30-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ae8c62fe2480dd61c532ccafdbce9b29dacc126fe8be0d9a927ca3e699b9491a"}, - {file = "SQLAlchemy-2.0.30-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2383146973a15435e4717f94c7509982770e3e54974c71f76500a0136f22810b"}, - {file = "SQLAlchemy-2.0.30-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8409de825f2c3b62ab15788635ccaec0c881c3f12a8af2b12ae4910a0a9aeef6"}, - {file = "SQLAlchemy-2.0.30-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0094c5dc698a5f78d3d1539853e8ecec02516b62b8223c970c86d44e7a80f6c7"}, - {file = "SQLAlchemy-2.0.30-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:edc16a50f5e1b7a06a2dcc1f2205b0b961074c123ed17ebda726f376a5ab0953"}, - {file = "SQLAlchemy-2.0.30-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:f7703c2010355dd28f53deb644a05fc30f796bd8598b43f0ba678878780b6e4c"}, - {file = "SQLAlchemy-2.0.30-cp39-cp39-win32.whl", hash = "sha256:1f9a727312ff6ad5248a4367358e2cf7e625e98b1028b1d7ab7b806b7d757513"}, - {file = "SQLAlchemy-2.0.30-cp39-cp39-win_amd64.whl", hash = "sha256:a0ef36b28534f2a5771191be6edb44cc2673c7b2edf6deac6562400288664221"}, - {file = "SQLAlchemy-2.0.30-py3-none-any.whl", hash = "sha256:7108d569d3990c71e26a42f60474b4c02c8586c4681af5fd67e51a044fdea86a"}, - {file = "SQLAlchemy-2.0.30.tar.gz", hash = "sha256:2b1708916730f4830bc69d6f49d37f7698b5bd7530aca7f04f785f8849e95255"}, + {file = "SQLAlchemy-2.0.31-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f2a213c1b699d3f5768a7272de720387ae0122f1becf0901ed6eaa1abd1baf6c"}, + {file = "SQLAlchemy-2.0.31-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9fea3d0884e82d1e33226935dac990b967bef21315cbcc894605db3441347443"}, + {file = "SQLAlchemy-2.0.31-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3ad7f221d8a69d32d197e5968d798217a4feebe30144986af71ada8c548e9fa"}, + {file = "SQLAlchemy-2.0.31-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f2bee229715b6366f86a95d497c347c22ddffa2c7c96143b59a2aa5cc9eebbc"}, + {file = "SQLAlchemy-2.0.31-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:cd5b94d4819c0c89280b7c6109c7b788a576084bf0a480ae17c227b0bc41e109"}, + {file = "SQLAlchemy-2.0.31-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:750900a471d39a7eeba57580b11983030517a1f512c2cb287d5ad0fcf3aebd58"}, + {file = "SQLAlchemy-2.0.31-cp310-cp310-win32.whl", hash = "sha256:7bd112be780928c7f493c1a192cd8c5fc2a2a7b52b790bc5a84203fb4381c6be"}, + {file = "SQLAlchemy-2.0.31-cp310-cp310-win_amd64.whl", hash = "sha256:5a48ac4d359f058474fadc2115f78a5cdac9988d4f99eae44917f36aa1476327"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f68470edd70c3ac3b6cd5c2a22a8daf18415203ca1b036aaeb9b0fb6f54e8298"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2e2c38c2a4c5c634fe6c3c58a789712719fa1bf9b9d6ff5ebfce9a9e5b89c1ca"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bd15026f77420eb2b324dcb93551ad9c5f22fab2c150c286ef1dc1160f110203"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2196208432deebdfe3b22185d46b08f00ac9d7b01284e168c212919891289396"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:352b2770097f41bff6029b280c0e03b217c2dcaddc40726f8f53ed58d8a85da4"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:56d51ae825d20d604583f82c9527d285e9e6d14f9a5516463d9705dab20c3740"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-win32.whl", hash = "sha256:6e2622844551945db81c26a02f27d94145b561f9d4b0c39ce7bfd2fda5776dac"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-win_amd64.whl", hash = "sha256:ccaf1b0c90435b6e430f5dd30a5aede4764942a695552eb3a4ab74ed63c5b8d3"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3b74570d99126992d4b0f91fb87c586a574a5872651185de8297c6f90055ae42"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6f77c4f042ad493cb8595e2f503c7a4fe44cd7bd59c7582fd6d78d7e7b8ec52c"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cd1591329333daf94467e699e11015d9c944f44c94d2091f4ac493ced0119449"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:74afabeeff415e35525bf7a4ecdab015f00e06456166a2eba7590e49f8db940e"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b9c01990d9015df2c6f818aa8f4297d42ee71c9502026bb074e713d496e26b67"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:66f63278db425838b3c2b1c596654b31939427016ba030e951b292e32b99553e"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-win32.whl", hash = "sha256:0b0f658414ee4e4b8cbcd4a9bb0fd743c5eeb81fc858ca517217a8013d282c96"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-win_amd64.whl", hash = "sha256:fa4b1af3e619b5b0b435e333f3967612db06351217c58bfb50cee5f003db2a5a"}, + {file = "SQLAlchemy-2.0.31-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:f43e93057cf52a227eda401251c72b6fbe4756f35fa6bfebb5d73b86881e59b0"}, + {file = "SQLAlchemy-2.0.31-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d337bf94052856d1b330d5fcad44582a30c532a2463776e1651bd3294ee7e58b"}, + {file = "SQLAlchemy-2.0.31-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c06fb43a51ccdff3b4006aafee9fcf15f63f23c580675f7734245ceb6b6a9e05"}, + {file = "SQLAlchemy-2.0.31-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:b6e22630e89f0e8c12332b2b4c282cb01cf4da0d26795b7eae16702a608e7ca1"}, + {file = "SQLAlchemy-2.0.31-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:79a40771363c5e9f3a77f0e28b3302801db08040928146e6808b5b7a40749c88"}, + {file = "SQLAlchemy-2.0.31-cp37-cp37m-win32.whl", hash = "sha256:501ff052229cb79dd4c49c402f6cb03b5a40ae4771efc8bb2bfac9f6c3d3508f"}, + {file = "SQLAlchemy-2.0.31-cp37-cp37m-win_amd64.whl", hash = "sha256:597fec37c382a5442ffd471f66ce12d07d91b281fd474289356b1a0041bdf31d"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:dc6d69f8829712a4fd799d2ac8d79bdeff651c2301b081fd5d3fe697bd5b4ab9"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:23b9fbb2f5dd9e630db70fbe47d963c7779e9c81830869bd7d137c2dc1ad05fb"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2a21c97efcbb9f255d5c12a96ae14da873233597dfd00a3a0c4ce5b3e5e79704"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:26a6a9837589c42b16693cf7bf836f5d42218f44d198f9343dd71d3164ceeeac"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:dc251477eae03c20fae8db9c1c23ea2ebc47331bcd73927cdcaecd02af98d3c3"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:2fd17e3bb8058359fa61248c52c7b09a97cf3c820e54207a50af529876451808"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-win32.whl", hash = "sha256:c76c81c52e1e08f12f4b6a07af2b96b9b15ea67ccdd40ae17019f1c373faa227"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-win_amd64.whl", hash = "sha256:4b600e9a212ed59355813becbcf282cfda5c93678e15c25a0ef896b354423238"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b6cf796d9fcc9b37011d3f9936189b3c8074a02a4ed0c0fbbc126772c31a6d4"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:78fe11dbe37d92667c2c6e74379f75746dc947ee505555a0197cfba9a6d4f1a4"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2fc47dc6185a83c8100b37acda27658fe4dbd33b7d5e7324111f6521008ab4fe"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a41514c1a779e2aa9a19f67aaadeb5cbddf0b2b508843fcd7bafdf4c6864005"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:afb6dde6c11ea4525318e279cd93c8734b795ac8bb5dda0eedd9ebaca7fa23f1"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:3f9faef422cfbb8fd53716cd14ba95e2ef655400235c3dfad1b5f467ba179c8c"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-win32.whl", hash = "sha256:fc6b14e8602f59c6ba893980bea96571dd0ed83d8ebb9c4479d9ed5425d562e9"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-win_amd64.whl", hash = "sha256:3cb8a66b167b033ec72c3812ffc8441d4e9f5f78f5e31e54dcd4c90a4ca5bebc"}, + {file = "SQLAlchemy-2.0.31-py3-none-any.whl", hash = "sha256:69f3e3c08867a8e4856e92d7afb618b95cdee18e0bc1647b77599722c9a28911"}, + {file = "SQLAlchemy-2.0.31.tar.gz", hash = "sha256:b607489dd4a54de56984a0c7656247504bd5523d9d0ba799aef59d4add009484"}, ] [package.dependencies] -greenlet = {version = "!=0.4.17", markers = "platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\""} +greenlet = {version = "!=0.4.17", markers = "python_version < \"3.13\" and (platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\")"} typing-extensions = ">=4.6.0" [package.extras] @@ -4748,4 +4752,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.12.2" -content-hash = "122fe6459adad8b8105d12739e4db159ae39fc32e969d02c7c24436ed3fb85b1" +content-hash = "951d69f5399c4dd7f38a21bd53299ad938bdf34f527f70fe9519c783ea86b899" diff --git a/pyproject.toml b/pyproject.toml index ff5d6a959..125bff5d0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,7 +46,7 @@ poetry-dotenv-plugin = "==0.2.0" psycopg2-binary = "==2.9.9" pyjwt = "==2.8.0" python-dotenv = "==1.0.1" -sqlalchemy = "==2.0.30" +sqlalchemy = "==2.0.31" werkzeug = "^3.0.3" faker = "^25.8.0" async-timeout = "^4.0.3" From 530e437565992f133b29dbbd35d742628745010e Mon Sep 17 00:00:00 2001 From: Carlo Costino Date: Mon, 24 Jun 2024 12:03:28 -0400 Subject: [PATCH 19/28] Increased production memory to 2 GB This changeset increases our memory for the API app and its corresponding workers to 2 GB each. Signed-off-by: Carlo Costino --- deploy-config/production.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy-config/production.yml b/deploy-config/production.yml index 18ed2a0c3..fc5be0451 100644 --- a/deploy-config/production.yml +++ b/deploy-config/production.yml @@ -1,8 +1,8 @@ env: production web_instances: 2 -web_memory: 1G +web_memory: 2G worker_instances: 1 -worker_memory: 1G +worker_memory: 2G scheduler_memory: 256M public_api_route: notify-api.app.cloud.gov admin_base_url: https://beta.notify.gov From 8a1851b61ccef5d653b8f224ce176d34709140a6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Jun 2024 21:07:59 +0000 Subject: [PATCH 20/28] Bump botocore from 1.34.128 to 1.34.131 Bumps [botocore](https://github.com/boto/botocore) from 1.34.128 to 1.34.131. - [Changelog](https://github.com/boto/botocore/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/botocore/compare/1.34.128...1.34.131) --- updated-dependencies: - dependency-name: botocore dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- poetry.lock | 16 ++++++++-------- pyproject.toml | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/poetry.lock b/poetry.lock index 47991d267..05563c31b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -204,17 +204,17 @@ tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "p [[package]] name = "awscli" -version = "1.33.10" +version = "1.33.13" description = "Universal Command Line Environment for AWS." optional = false python-versions = ">=3.8" files = [ - {file = "awscli-1.33.10-py3-none-any.whl", hash = "sha256:f3cdd988c4fd1cf25e8fe16fccc514147469ed102b43c128e29374d48337d7a0"}, - {file = "awscli-1.33.10.tar.gz", hash = "sha256:0a5b4b5555be2c7324bf9e748bc24c17c95ef1b070e8f324354ef8747ce984e9"}, + {file = "awscli-1.33.13-py3-none-any.whl", hash = "sha256:ec5846b00a045445446fa8287ae57be84122a26aee8d7d81e1e5dc1d2ad91c6b"}, + {file = "awscli-1.33.13.tar.gz", hash = "sha256:bad4402c4a0ff825a59a43e76e0072152497f4daf7f62c9376fe6e0014fa2a9b"}, ] [package.dependencies] -botocore = "1.34.128" +botocore = "1.34.131" colorama = ">=0.2.5,<0.4.7" docutils = ">=0.10,<0.17" PyYAML = ">=3.10,<6.1" @@ -422,13 +422,13 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.34.128" +version = "1.34.131" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">=3.8" files = [ - {file = "botocore-1.34.128-py3-none-any.whl", hash = "sha256:db67fda136c372ab3fa432580c819c89ba18d28a6152a4d2a7ea40d44082892e"}, - {file = "botocore-1.34.128.tar.gz", hash = "sha256:8d8e03f7c8c080ecafda72036eb3b482d649f8417c90b5dca33b7c2c47adb0c9"}, + {file = "botocore-1.34.131-py3-none-any.whl", hash = "sha256:13b011d7b206ce00727dcee26548fa3b550db9046d5a0e90ac25a6e6c8fde6ef"}, + {file = "botocore-1.34.131.tar.gz", hash = "sha256:502ddafe1d627fcf1e4c007c86454e5dd011dba7c58bd8e8a5368a79f3e387dc"}, ] [package.dependencies] @@ -4752,4 +4752,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.12.2" -content-hash = "951d69f5399c4dd7f38a21bd53299ad938bdf34f527f70fe9519c783ea86b899" +content-hash = "24b12358e9f7291ab3df1d8b2b777fcfdc9ace00658fdd498324359d3aeada67" diff --git a/pyproject.toml b/pyproject.toml index 125bff5d0..5ac63debe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ alembic = "==1.13.1" amqp = "==5.2.0" beautifulsoup4 = "==4.12.3" boto3 = "^1.34.128" -botocore = "^1.34.128" +botocore = "^1.34.131" cachetools = "==5.3.3" celery = {version = "==5.4.0", extras = ["redis"]} certifi = ">=2022.12.7" From 3b152b260ba28403c7d13f8928c147fa8e6178b5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Jun 2024 21:58:43 +0000 Subject: [PATCH 21/28] Bump boto3 from 1.34.128 to 1.34.131 Bumps [boto3](https://github.com/boto/boto3) from 1.34.128 to 1.34.131. - [Release notes](https://github.com/boto/boto3/releases) - [Changelog](https://github.com/boto/boto3/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/boto3/compare/1.34.128...1.34.131) --- updated-dependencies: - dependency-name: boto3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- poetry.lock | 10 +++++----- pyproject.toml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/poetry.lock b/poetry.lock index 05563c31b..39f730b5a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -403,17 +403,17 @@ files = [ [[package]] name = "boto3" -version = "1.34.128" +version = "1.34.131" description = "The AWS SDK for Python" optional = false python-versions = ">=3.8" files = [ - {file = "boto3-1.34.128-py3-none-any.whl", hash = "sha256:a048ff980a81cd652724a73bc496c519b336fabe19cc8bfc6c53b2ff6eb22c7b"}, - {file = "boto3-1.34.128.tar.gz", hash = "sha256:43a6e99f53a8d34b3b4dbe424dbcc6b894350dc41a85b0af7c7bc24a7ec2cead"}, + {file = "boto3-1.34.131-py3-none-any.whl", hash = "sha256:05e388cb937e82be70bfd7eb0c84cf8011ff35cf582a593873ac21675268683b"}, + {file = "boto3-1.34.131.tar.gz", hash = "sha256:dab8f72a6c4e62b4fd70da09e08a6b2a65ea2115b27dd63737142005776ef216"}, ] [package.dependencies] -botocore = ">=1.34.128,<1.35.0" +botocore = ">=1.34.131,<1.35.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.10.0,<0.11.0" @@ -4752,4 +4752,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.12.2" -content-hash = "24b12358e9f7291ab3df1d8b2b777fcfdc9ace00658fdd498324359d3aeada67" +content-hash = "515b0cbed0890a425c2578789b1f46cce3011088fe275e16ef802ddb2597f7a1" diff --git a/pyproject.toml b/pyproject.toml index 5ac63debe..a3639760a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ python = "^3.12.2" alembic = "==1.13.1" amqp = "==5.2.0" beautifulsoup4 = "==4.12.3" -boto3 = "^1.34.128" +boto3 = "^1.34.131" botocore = "^1.34.131" cachetools = "==5.3.3" celery = {version = "==5.4.0", extras = ["redis"]} From f69f52b91a9eaf4663bcaf39ecba69e937a5616d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 25 Jun 2024 01:08:57 +0000 Subject: [PATCH 22/28] Bump setuptools from 70.0.0 to 70.1.0 Bumps [setuptools](https://github.com/pypa/setuptools) from 70.0.0 to 70.1.0. - [Release notes](https://github.com/pypa/setuptools/releases) - [Changelog](https://github.com/pypa/setuptools/blob/main/NEWS.rst) - [Commits](https://github.com/pypa/setuptools/compare/v70.0.0...v70.1.0) --- updated-dependencies: - dependency-name: setuptools dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- poetry.lock | 10 +++++----- pyproject.toml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/poetry.lock b/poetry.lock index 39f730b5a..91e1c0046 100644 --- a/poetry.lock +++ b/poetry.lock @@ -4021,18 +4021,18 @@ jeepney = ">=0.6" [[package]] name = "setuptools" -version = "70.0.0" +version = "70.1.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-70.0.0-py3-none-any.whl", hash = "sha256:54faa7f2e8d2d11bcd2c07bed282eef1046b5c080d1c32add737d7b5817b1ad4"}, - {file = "setuptools-70.0.0.tar.gz", hash = "sha256:f211a66637b8fa059bb28183da127d4e86396c991a942b028c6650d4319c3fd0"}, + {file = "setuptools-70.1.0-py3-none-any.whl", hash = "sha256:d9b8b771455a97c8a9f3ab3448ebe0b29b5e105f1228bba41028be116985a267"}, + {file = "setuptools-70.1.0.tar.gz", hash = "sha256:01a1e793faa5bd89abc851fa15d0a0db26f160890c7102cd8dce643e886b47f5"}, ] [package.extras] docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.1)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test", "mypy (==1.10.0)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.1)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.3.2)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] [[package]] name = "shapely" @@ -4752,4 +4752,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.12.2" -content-hash = "515b0cbed0890a425c2578789b1f46cce3011088fe275e16ef802ddb2597f7a1" +content-hash = "2f916e3cfb589391596ef0ffa9dbeb52c362e10d1767d8ddbf4e9ef9c0ceedfe" diff --git a/pyproject.toml b/pyproject.toml index a3639760a..d3980b542 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -102,7 +102,7 @@ pytest-cov = "^5.0.0" pytest-xdist = "^3.5.0" radon = "^6.0.1" requests-mock = "^1.11.0" -setuptools = "^70.0.0" +setuptools = "^70.1.0" sqlalchemy-utils = "^0.41.2" vulture = "^2.10" detect-secrets = "^1.5.0" From 28d77b12ed60ced3402d5d9ffc72e87dd364caa1 Mon Sep 17 00:00:00 2001 From: John Skiles Skinner Date: Tue, 25 Jun 2024 10:04:21 -0700 Subject: [PATCH 23/28] Upgrade major verion 41 to 44 of tj-actions --- .github/workflows/deploy.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 104fa1521..4cbd88acc 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -23,7 +23,7 @@ jobs: - name: Check for changes to Terraform id: changed-terraform-files - uses: tj-actions/changed-files@v41 + uses: tj-actions/changed-files@v44 with: files: | terraform/staging @@ -81,7 +81,7 @@ jobs: - name: Check for changes to templates.json id: changed-templates - uses: tj-actions/changed-files@v41 + uses: tj-actions/changed-files@v44 with: files: | app/config_files/templates.json @@ -91,7 +91,7 @@ jobs: - name: Check for changes to egress config id: changed-egress-config - uses: tj-actions/changed-files@v41 + uses: tj-actions/changed-files@v44 with: files: | deploy-config/egress_proxy/notify-api-staging.*.acl From 8aa3683e7397c7655d0118efb26e4af3ece60b2e Mon Sep 17 00:00:00 2001 From: John Skiles Skinner Date: Tue, 25 Jun 2024 10:59:59 -0700 Subject: [PATCH 24/28] Upgrade from v41 to v44 in demo and prod deploy workflows --- .github/workflows/deploy-demo.yml | 6 +++--- .github/workflows/deploy-prod.yml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/deploy-demo.yml b/.github/workflows/deploy-demo.yml index 945540b19..b562131db 100644 --- a/.github/workflows/deploy-demo.yml +++ b/.github/workflows/deploy-demo.yml @@ -18,7 +18,7 @@ jobs: - name: Check for changes to Terraform id: changed-terraform-files - uses: tj-actions/changed-files@v41 + uses: tj-actions/changed-files@v44 with: files: | terraform/demo @@ -76,7 +76,7 @@ jobs: - name: Check for changes to templates.json id: changed-templates - uses: tj-actions/changed-files@v41 + uses: tj-actions/changed-files@v44 with: files: | app/config_files/templates.json @@ -86,7 +86,7 @@ jobs: - name: Check for changes to egress config id: changed-egress-config - uses: tj-actions/changed-files@v41 + uses: tj-actions/changed-files@v44 with: files: | deploy-config/egress_proxy/notify-api-demo.*.acl diff --git a/.github/workflows/deploy-prod.yml b/.github/workflows/deploy-prod.yml index 20d452b4a..fa5036457 100644 --- a/.github/workflows/deploy-prod.yml +++ b/.github/workflows/deploy-prod.yml @@ -22,7 +22,7 @@ jobs: - name: Check for changes to Terraform id: changed-terraform-files - uses: tj-actions/changed-files@v41 + uses: tj-actions/changed-files@v44 with: files: | terraform/production @@ -80,7 +80,7 @@ jobs: - name: Check for changes to templates.json id: changed-templates - uses: tj-actions/changed-files@v41 + uses: tj-actions/changed-files@v44 with: files: | app/config_files/templates.json @@ -90,7 +90,7 @@ jobs: - name: Check for changes to egress config id: changed-egress-config - uses: tj-actions/changed-files@v41 + uses: tj-actions/changed-files@v44 with: files: | deploy-config/egress_proxy/notify-api-production.*.acl From a07a79014cdc17ddbec83ff2e4ad583a5513a960 Mon Sep 17 00:00:00 2001 From: John Skiles Skinner Date: Tue, 25 Jun 2024 12:21:43 -0700 Subject: [PATCH 25/28] Remove old version of Redis from Staging (and Sandbox) --- terraform/sandbox/main.tf | 9 --------- terraform/staging/main.tf | 9 --------- 2 files changed, 18 deletions(-) diff --git a/terraform/sandbox/main.tf b/terraform/sandbox/main.tf index 319b273ba..380131370 100644 --- a/terraform/sandbox/main.tf +++ b/terraform/sandbox/main.tf @@ -21,15 +21,6 @@ module "database" { rds_plan_name = "micro-psql" } -module "redis" { # default v6.2; delete after v7.0 resource is bound - source = "github.com/GSA-TTS/terraform-cloudgov//redis?ref=v1.0.0" - - cf_org_name = local.cf_org_name - cf_space_name = local.cf_space_name - name = "${local.app_name}-redis-${local.env}" - redis_plan_name = "redis-dev" -} - module "redis-v70" { source = "github.com/GSA-TTS/terraform-cloudgov//redis?ref=v1.0.0" diff --git a/terraform/staging/main.tf b/terraform/staging/main.tf index 3cc0358a5..4fdbf9e38 100644 --- a/terraform/staging/main.tf +++ b/terraform/staging/main.tf @@ -21,15 +21,6 @@ module "database" { rds_plan_name = "micro-psql" } -module "redis" { # default v6.2; delete after v7.0 resource is bound - source = "github.com/GSA-TTS/terraform-cloudgov//redis?ref=v1.0.0" - - cf_org_name = local.cf_org_name - cf_space_name = local.cf_space_name - name = "${local.app_name}-redis-${local.env}" - redis_plan_name = "redis-dev" -} - module "redis-v70" { source = "github.com/GSA-TTS/terraform-cloudgov//redis?ref=v1.0.0" From f70ab54f4a5312a14d9b92e87b526d96470a571f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 25 Jun 2024 21:21:12 +0000 Subject: [PATCH 26/28] Bump botocore from 1.34.131 to 1.34.133 Bumps [botocore](https://github.com/boto/botocore) from 1.34.131 to 1.34.133. - [Changelog](https://github.com/boto/botocore/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/botocore/compare/1.34.131...1.34.133) --- updated-dependencies: - dependency-name: botocore dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- poetry.lock | 16 ++++++++-------- pyproject.toml | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/poetry.lock b/poetry.lock index 91e1c0046..e0da75173 100644 --- a/poetry.lock +++ b/poetry.lock @@ -204,17 +204,17 @@ tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "p [[package]] name = "awscli" -version = "1.33.13" +version = "1.33.15" description = "Universal Command Line Environment for AWS." optional = false python-versions = ">=3.8" files = [ - {file = "awscli-1.33.13-py3-none-any.whl", hash = "sha256:ec5846b00a045445446fa8287ae57be84122a26aee8d7d81e1e5dc1d2ad91c6b"}, - {file = "awscli-1.33.13.tar.gz", hash = "sha256:bad4402c4a0ff825a59a43e76e0072152497f4daf7f62c9376fe6e0014fa2a9b"}, + {file = "awscli-1.33.15-py3-none-any.whl", hash = "sha256:5a8d7e68a4cf68afc9d9ba4bef511526eb71027360f95a1080d39158bc930083"}, + {file = "awscli-1.33.15.tar.gz", hash = "sha256:54a8089edb6756da46addcfcd56fdca21307a121216a81ef542e17b284cbe9c9"}, ] [package.dependencies] -botocore = "1.34.131" +botocore = "1.34.133" colorama = ">=0.2.5,<0.4.7" docutils = ">=0.10,<0.17" PyYAML = ">=3.10,<6.1" @@ -422,13 +422,13 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.34.131" +version = "1.34.133" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">=3.8" files = [ - {file = "botocore-1.34.131-py3-none-any.whl", hash = "sha256:13b011d7b206ce00727dcee26548fa3b550db9046d5a0e90ac25a6e6c8fde6ef"}, - {file = "botocore-1.34.131.tar.gz", hash = "sha256:502ddafe1d627fcf1e4c007c86454e5dd011dba7c58bd8e8a5368a79f3e387dc"}, + {file = "botocore-1.34.133-py3-none-any.whl", hash = "sha256:f269dad8e17432d2527b97ed9f1fd30ec8dc705f8b818957170d1af484680ef2"}, + {file = "botocore-1.34.133.tar.gz", hash = "sha256:5ea609aa4831a6589e32eef052a359ad8d7311733b4d86a9d35dab4bd3ec80ff"}, ] [package.dependencies] @@ -4752,4 +4752,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.12.2" -content-hash = "2f916e3cfb589391596ef0ffa9dbeb52c362e10d1767d8ddbf4e9ef9c0ceedfe" +content-hash = "e6a0bdbd3ef3112785636ccf65600188c45d7785d08d7c743b840b74a2274dff" diff --git a/pyproject.toml b/pyproject.toml index d3980b542..2ee6cfca0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ alembic = "==1.13.1" amqp = "==5.2.0" beautifulsoup4 = "==4.12.3" boto3 = "^1.34.131" -botocore = "^1.34.131" +botocore = "^1.34.133" cachetools = "==5.3.3" celery = {version = "==5.4.0", extras = ["redis"]} certifi = ">=2022.12.7" From edc002ea3a6adcec141b0223f6bc2ef8a021cf49 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Jun 2024 02:12:44 +0000 Subject: [PATCH 27/28] Bump setuptools from 70.1.0 to 70.1.1 Bumps [setuptools](https://github.com/pypa/setuptools) from 70.1.0 to 70.1.1. - [Release notes](https://github.com/pypa/setuptools/releases) - [Changelog](https://github.com/pypa/setuptools/blob/main/NEWS.rst) - [Commits](https://github.com/pypa/setuptools/compare/v70.1.0...v70.1.1) --- updated-dependencies: - dependency-name: setuptools dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- poetry.lock | 8 ++++---- pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index e0da75173..1c5be2257 100644 --- a/poetry.lock +++ b/poetry.lock @@ -4021,13 +4021,13 @@ jeepney = ">=0.6" [[package]] name = "setuptools" -version = "70.1.0" +version = "70.1.1" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-70.1.0-py3-none-any.whl", hash = "sha256:d9b8b771455a97c8a9f3ab3448ebe0b29b5e105f1228bba41028be116985a267"}, - {file = "setuptools-70.1.0.tar.gz", hash = "sha256:01a1e793faa5bd89abc851fa15d0a0db26f160890c7102cd8dce643e886b47f5"}, + {file = "setuptools-70.1.1-py3-none-any.whl", hash = "sha256:a58a8fde0541dab0419750bcc521fbdf8585f6e5cb41909df3a472ef7b81ca95"}, + {file = "setuptools-70.1.1.tar.gz", hash = "sha256:937a48c7cdb7a21eb53cd7f9b59e525503aa8abaf3584c730dc5f7a5bec3a650"}, ] [package.extras] @@ -4752,4 +4752,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.12.2" -content-hash = "e6a0bdbd3ef3112785636ccf65600188c45d7785d08d7c743b840b74a2274dff" +content-hash = "b1b4bfbfdc1f5cc9ae9d090f35b235a62e9dbabc683a5b5a1d0d414605219b48" diff --git a/pyproject.toml b/pyproject.toml index 2ee6cfca0..2ea004520 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -102,7 +102,7 @@ pytest-cov = "^5.0.0" pytest-xdist = "^3.5.0" radon = "^6.0.1" requests-mock = "^1.11.0" -setuptools = "^70.1.0" +setuptools = "^70.1.1" sqlalchemy-utils = "^0.41.2" vulture = "^2.10" detect-secrets = "^1.5.0" From 063ec69f70a420c4b9140706213b8f47d5012256 Mon Sep 17 00:00:00 2001 From: John Skiles Skinner Date: Wed, 26 Jun 2024 11:09:50 -0700 Subject: [PATCH 28/28] Update name of Redis to include v70 --- manifest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifest.yml b/manifest.yml index e026ea09b..a8a3e7f2b 100644 --- a/manifest.yml +++ b/manifest.yml @@ -11,7 +11,7 @@ applications: services: - notify-api-rds-((env)) - - notify-api-redis-((env)) + - notify-api-redis-v70-((env)) - notify-api-csv-upload-bucket-((env)) - name: notify-api-ses-((env)) parameters: