diff --git a/.github/workflows/deploy-demo.yml b/.github/workflows/deploy-demo.yml index 945540b19..b562131db 100644 --- a/.github/workflows/deploy-demo.yml +++ b/.github/workflows/deploy-demo.yml @@ -18,7 +18,7 @@ jobs: - name: Check for changes to Terraform id: changed-terraform-files - uses: tj-actions/changed-files@v41 + uses: tj-actions/changed-files@v44 with: files: | terraform/demo @@ -76,7 +76,7 @@ jobs: - name: Check for changes to templates.json id: changed-templates - uses: tj-actions/changed-files@v41 + uses: tj-actions/changed-files@v44 with: files: | app/config_files/templates.json @@ -86,7 +86,7 @@ jobs: - name: Check for changes to egress config id: changed-egress-config - uses: tj-actions/changed-files@v41 + uses: tj-actions/changed-files@v44 with: files: | deploy-config/egress_proxy/notify-api-demo.*.acl diff --git a/.github/workflows/deploy-prod.yml b/.github/workflows/deploy-prod.yml index 20d452b4a..fa5036457 100644 --- a/.github/workflows/deploy-prod.yml +++ b/.github/workflows/deploy-prod.yml @@ -22,7 +22,7 @@ jobs: - name: Check for changes to Terraform id: changed-terraform-files - uses: tj-actions/changed-files@v41 + uses: tj-actions/changed-files@v44 with: files: | terraform/production @@ -80,7 +80,7 @@ jobs: - name: Check for changes to templates.json id: changed-templates - uses: tj-actions/changed-files@v41 + uses: tj-actions/changed-files@v44 with: files: | app/config_files/templates.json @@ -90,7 +90,7 @@ jobs: - name: Check for changes to egress config id: changed-egress-config - uses: tj-actions/changed-files@v41 + uses: tj-actions/changed-files@v44 with: files: | deploy-config/egress_proxy/notify-api-production.*.acl diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 104fa1521..4cbd88acc 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -23,7 +23,7 @@ jobs: - name: Check for changes to Terraform id: changed-terraform-files - uses: tj-actions/changed-files@v41 + uses: tj-actions/changed-files@v44 with: files: | terraform/staging @@ -81,7 +81,7 @@ jobs: - name: Check for changes to templates.json id: changed-templates - uses: tj-actions/changed-files@v41 + uses: tj-actions/changed-files@v44 with: files: | app/config_files/templates.json @@ -91,7 +91,7 @@ jobs: - name: Check for changes to egress config id: changed-egress-config - uses: tj-actions/changed-files@v41 + uses: tj-actions/changed-files@v44 with: files: | deploy-config/egress_proxy/notify-api-staging.*.acl diff --git a/app/celery/tasks.py b/app/celery/tasks.py index 3f3fb2196..ecaa8479b 100644 --- a/app/celery/tasks.py +++ b/app/celery/tasks.py @@ -26,7 +26,7 @@ from app.notifications.process_notifications import persist_notification from app.notifications.validators import check_service_over_total_message_limit from app.serialised_models import SerialisedService, SerialisedTemplate from app.service.utils import service_allowed_to_send_to -from app.utils import DATETIME_FORMAT, hilite, scrub, utc_now +from app.utils import DATETIME_FORMAT, hilite, utc_now from notifications_utils.recipients import RecipientCSV @@ -193,7 +193,7 @@ def save_sms(self, service_id, notification_id, encrypted_notification, sender_i if not service_allowed_to_send_to(notification["to"], service, KeyType.NORMAL): current_app.logger.info( hilite( - scrub(f"service not allowed to send to {notification['to']}, aborting") + f"service not allowed to send for job_id {notification.get('job', None)}, aborting" ) ) current_app.logger.debug( @@ -226,8 +226,11 @@ def save_sms(self, service_id, notification_id, encrypted_notification, sender_i ) # Kick off sns process in provider_tasks.py + sn = saved_notification current_app.logger.info( - hilite(scrub(f"Going to deliver sms for recipient: {notification['to']}")) + hilite( + f"Deliver sms for job_id: {sn.job_id} row_number: {sn.job_row_number}" + ) ) provider_tasks.deliver_sms.apply_async( [str(saved_notification.id)], queue=QueueNames.SEND_SMS diff --git a/app/clients/cloudwatch/aws_cloudwatch.py b/app/clients/cloudwatch/aws_cloudwatch.py index 14d8513c5..36bcf5dca 100644 --- a/app/clients/cloudwatch/aws_cloudwatch.py +++ b/app/clients/cloudwatch/aws_cloudwatch.py @@ -9,7 +9,7 @@ from flask import current_app from app.clients import AWS_CLIENT_CONFIG, Client from app.cloudfoundry_config import cloud_config from app.exceptions import NotificationTechnicalFailureException -from app.utils import hilite, scrub, utc_now +from app.utils import hilite, utc_now class AwsCloudwatchClient(Client): @@ -124,7 +124,14 @@ class AwsCloudwatchClient(Client): self.warn_if_dev_is_opted_out( message["delivery"]["providerResponse"], notification_id ) - current_app.logger.info(hilite(scrub(f"DELIVERED: {message}"))) + # Here we map the answer from aws to the message_id. + # Previously, in send_to_providers, we mapped the job_id and row number + # to the message id. And on the admin side we mapped the csv filename + # to the job_id. So by tracing through all the logs we can go: + # filename->job_id->message_id->what really happened + current_app.logger.info( + hilite(f"DELIVERED: {message} for message_id {message_id}") + ) return ( "success", message["delivery"]["providerResponse"], @@ -142,7 +149,9 @@ class AwsCloudwatchClient(Client): message["delivery"]["providerResponse"], notification_id ) - current_app.logger.info(hilite(scrub(f"FAILED: {message}"))) + current_app.logger.info( + hilite(f"FAILED: {message} for message_id {message_id}") + ) return ( "failure", message["delivery"]["providerResponse"], diff --git a/app/delivery/send_to_providers.py b/app/delivery/send_to_providers.py index a347b89ad..c0d32bf86 100644 --- a/app/delivery/send_to_providers.py +++ b/app/delivery/send_to_providers.py @@ -13,7 +13,7 @@ from app.dao.provider_details_dao import get_provider_details_by_notification_ty from app.enums import BrandType, KeyType, NotificationStatus, NotificationType from app.exceptions import NotificationTechnicalFailureException from app.serialised_models import SerialisedService, SerialisedTemplate -from app.utils import hilite, scrub, utc_now +from app.utils import hilite, utc_now from notifications_utils.template import ( HTMLEmailTemplate, PlainTextEmailTemplate, @@ -89,7 +89,7 @@ def send_sms_to_provider(notification): except Exception: # It is our 2facode, maybe key = f"2facode-{notification.id}".replace(" ", "") - recipient = redis_store.raw_get(key) + recipient = redis_store.get(key) if recipient: recipient = recipient.decode("utf-8") @@ -113,15 +113,18 @@ def send_sms_to_provider(notification): message_id = provider.send_sms(**send_sms_kwargs) current_app.logger.info(f"got message_id {message_id}") except Exception as e: - msg = f"FAILED sending message for this recipient: {recipient} to sms" - current_app.logger.error(hilite(scrub(f"{msg} {e}"))) + n = notification + msg = f"FAILED send to sms, job_id: {n.job_id} row_number {n.job_row_number} message_id {message_id}" + current_app.logger.error(hilite(f"{msg} {e}")) notification.billable_units = template.fragment_count dao_update_notification(notification) raise e else: - msg = f"Sending message for this recipient: {recipient} to sms" - current_app.logger.info(hilite(scrub(msg))) + # Here we map the job_id and row number to the aws message_id + n = notification + msg = f"Send to aws for job_id {n.job_id} row_number {n.job_row_number} message_id {message_id}" + current_app.logger.info(hilite(msg)) notification.billable_units = template.fragment_count update_notification_to_sending(notification, provider) return message_id diff --git a/app/notifications/process_notifications.py b/app/notifications/process_notifications.py index 7c97cd3ba..9d38ef1f2 100644 --- a/app/notifications/process_notifications.py +++ b/app/notifications/process_notifications.py @@ -12,7 +12,7 @@ from app.dao.notifications_dao import ( from app.enums import KeyType, NotificationStatus, NotificationType from app.errors import BadRequestError from app.models import Notification -from app.utils import hilite, scrub, utc_now +from app.utils import hilite, utc_now from notifications_utils.recipients import ( format_email_address, get_international_phone_info, @@ -112,7 +112,7 @@ def persist_notification( ) current_app.logger.info( hilite( - scrub(f"Persisting notification with recipient {formatted_recipient}") + f"Persisting notification with job_id: {job_id} row_number: {job_row_number}" ) ) recipient_info = get_international_phone_info(formatted_recipient) diff --git a/app/user/rest.py b/app/user/rest.py index 760334841..049549f2c 100644 --- a/app/user/rest.py +++ b/app/user/rest.py @@ -361,7 +361,7 @@ def create_2fa_code( saved_notification.personalisation = personalisation key = f"2facode-{saved_notification.id}".replace(" ", "") recipient = str(recipient) - redis_store.raw_set(key, recipient, ex=60 * 60) + redis_store.set(key, recipient, ex=60 * 60) # Assume that we never want to observe the Notify service's research mode # setting for this notification - we still need to be able to log into the diff --git a/app/utils.py b/app/utils.py index b23a000d1..e286d6539 100644 --- a/app/utils.py +++ b/app/utils.py @@ -1,4 +1,3 @@ -import re from datetime import datetime, timedelta, timezone from flask import url_for @@ -145,15 +144,3 @@ def naive_utcnow(): def utc_now(): return naive_utcnow() - - -def scrub(msg): - # Eventually we want to scrub all messages in all logs for phone numbers - # and email addresses, masking them. Ultimately this will probably get - # refactored into a 'SafeLogger' subclass or something, but let's start here - # with phones. - phones = re.findall("(?:\\+ *)?\\d[\\d\\- ]{7,}\\d", msg) - phones = [phone.replace("-", "").replace(" ", "") for phone in phones] - for phone in phones: - msg = msg.replace(phone, f"1XXXXX{phone[-5:]}") - return msg diff --git a/deploy-config/production.yml b/deploy-config/production.yml index 18ed2a0c3..fc5be0451 100644 --- a/deploy-config/production.yml +++ b/deploy-config/production.yml @@ -1,8 +1,8 @@ env: production web_instances: 2 -web_memory: 1G +web_memory: 2G worker_instances: 1 -worker_memory: 1G +worker_memory: 2G scheduler_memory: 256M public_api_route: notify-api.app.cloud.gov admin_base_url: https://beta.notify.gov diff --git a/manifest.yml b/manifest.yml index e026ea09b..a8a3e7f2b 100644 --- a/manifest.yml +++ b/manifest.yml @@ -11,7 +11,7 @@ applications: services: - notify-api-rds-((env)) - - notify-api-redis-((env)) + - notify-api-redis-v70-((env)) - notify-api-csv-upload-bucket-((env)) - name: notify-api-ses-((env)) parameters: diff --git a/notifications_utils/clients/redis/redis_client.py b/notifications_utils/clients/redis/redis_client.py index 6e81d85bf..1723dd2c1 100644 --- a/notifications_utils/clients/redis/redis_client.py +++ b/notifications_utils/clients/redis/redis_client.py @@ -133,19 +133,13 @@ class RedisClient: else: return False - def raw_set(self, key, value, ex=None, px=None, nx=False, xx=False): - self.redis_store.set(key, value, ex, px, nx, xx) - def set( self, key, value, ex=None, px=None, nx=False, xx=False, raise_exception=False ): key = prepare_value(key) value = prepare_value(value) if self.active: - try: - self.redis_store.set(key, value, ex, px, nx, xx) - except Exception as e: - self.__handle_exception(e, raise_exception, "set", key) + self.redis_store.set(key, value, ex, px, nx, xx) def incr(self, key, raise_exception=False): key = prepare_value(key) @@ -155,16 +149,10 @@ class RedisClient: except Exception as e: self.__handle_exception(e, raise_exception, "incr", key) - def raw_get(self, key): - return self.redis_store.get(key) - def get(self, key, raise_exception=False): key = prepare_value(key) if self.active: - try: - return self.redis_store.get(key) - except Exception as e: - self.__handle_exception(e, raise_exception, "get", key) + return self.redis_store.get(key) return None diff --git a/notifications_utils/logging.py b/notifications_utils/logging.py index 6a209cdd3..9f5a98c2e 100644 --- a/notifications_utils/logging.py +++ b/notifications_utils/logging.py @@ -1,5 +1,6 @@ import logging import logging.handlers +import re import sys from itertools import product @@ -69,6 +70,7 @@ def configure_handler(handler, app, formatter): handler.addFilter(AppNameFilter(app.config["NOTIFY_APP_NAME"])) handler.addFilter(RequestIdFilter()) handler.addFilter(ServiceIdFilter()) + handler.addFilter(PIIFilter()) return handler @@ -115,6 +117,36 @@ class ServiceIdFilter(logging.Filter): return record +class PIIFilter(logging.Filter): + def scrub(self, msg): + # Eventually we want to scrub all messages in all logs for phone numbers + # and email addresses, masking them. Ultimately this will probably get + # refactored into a 'SafeLogger' subclass or something, but let's start here + # with phones. + + # Sometimes just an exception object is passed in for the message, skip those. + if not isinstance(msg, str): + return msg + phones = re.findall("(?:\\+ *)?\\d[\\d\\- ]{7,}\\d", msg) + + phones = [phone.replace("-", "").replace(" ", "") for phone in phones] + for phone in phones: + msg = msg.replace(phone, "1XXXXXXXXXX") + + emails = re.findall( + r"[\w\.-]+@[\w\.-]+", msg + ) # ['alice@google.com', 'bob@abc.com'] + for email in emails: + # do something with each found email string + masked_email = "XXXXX@XXXXXXX" + msg = msg.replace(email, masked_email) + return msg + + def filter(self, record): + record.msg = self.scrub(record.msg) + return record + + class JSONFormatter(BaseJSONFormatter): def process_log_record(self, log_record): rename_map = { diff --git a/poetry.lock b/poetry.lock index 29222296c..28d284d0b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -204,17 +204,17 @@ tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "p [[package]] name = "awscli" -version = "1.33.10" +version = "1.33.15" description = "Universal Command Line Environment for AWS." optional = false python-versions = ">=3.8" files = [ - {file = "awscli-1.33.10-py3-none-any.whl", hash = "sha256:f3cdd988c4fd1cf25e8fe16fccc514147469ed102b43c128e29374d48337d7a0"}, - {file = "awscli-1.33.10.tar.gz", hash = "sha256:0a5b4b5555be2c7324bf9e748bc24c17c95ef1b070e8f324354ef8747ce984e9"}, + {file = "awscli-1.33.15-py3-none-any.whl", hash = "sha256:5a8d7e68a4cf68afc9d9ba4bef511526eb71027360f95a1080d39158bc930083"}, + {file = "awscli-1.33.15.tar.gz", hash = "sha256:54a8089edb6756da46addcfcd56fdca21307a121216a81ef542e17b284cbe9c9"}, ] [package.dependencies] -botocore = "1.34.128" +botocore = "1.34.133" colorama = ">=0.2.5,<0.4.7" docutils = ">=0.10,<0.17" PyYAML = ">=3.10,<6.1" @@ -403,17 +403,17 @@ files = [ [[package]] name = "boto3" -version = "1.34.128" +version = "1.34.131" description = "The AWS SDK for Python" optional = false python-versions = ">=3.8" files = [ - {file = "boto3-1.34.128-py3-none-any.whl", hash = "sha256:a048ff980a81cd652724a73bc496c519b336fabe19cc8bfc6c53b2ff6eb22c7b"}, - {file = "boto3-1.34.128.tar.gz", hash = "sha256:43a6e99f53a8d34b3b4dbe424dbcc6b894350dc41a85b0af7c7bc24a7ec2cead"}, + {file = "boto3-1.34.131-py3-none-any.whl", hash = "sha256:05e388cb937e82be70bfd7eb0c84cf8011ff35cf582a593873ac21675268683b"}, + {file = "boto3-1.34.131.tar.gz", hash = "sha256:dab8f72a6c4e62b4fd70da09e08a6b2a65ea2115b27dd63737142005776ef216"}, ] [package.dependencies] -botocore = ">=1.34.128,<1.35.0" +botocore = ">=1.34.131,<1.35.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.10.0,<0.11.0" @@ -422,13 +422,13 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.34.128" +version = "1.34.133" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">=3.8" files = [ - {file = "botocore-1.34.128-py3-none-any.whl", hash = "sha256:db67fda136c372ab3fa432580c819c89ba18d28a6152a4d2a7ea40d44082892e"}, - {file = "botocore-1.34.128.tar.gz", hash = "sha256:8d8e03f7c8c080ecafda72036eb3b482d649f8417c90b5dca33b7c2c47adb0c9"}, + {file = "botocore-1.34.133-py3-none-any.whl", hash = "sha256:f269dad8e17432d2527b97ed9f1fd30ec8dc705f8b818957170d1af484680ef2"}, + {file = "botocore-1.34.133.tar.gz", hash = "sha256:5ea609aa4831a6589e32eef052a359ad8d7311733b4d86a9d35dab4bd3ec80ff"}, ] [package.dependencies] @@ -4017,18 +4017,18 @@ jeepney = ">=0.6" [[package]] name = "setuptools" -version = "70.0.0" +version = "70.1.1" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-70.0.0-py3-none-any.whl", hash = "sha256:54faa7f2e8d2d11bcd2c07bed282eef1046b5c080d1c32add737d7b5817b1ad4"}, - {file = "setuptools-70.0.0.tar.gz", hash = "sha256:f211a66637b8fa059bb28183da127d4e86396c991a942b028c6650d4319c3fd0"}, + {file = "setuptools-70.1.1-py3-none-any.whl", hash = "sha256:a58a8fde0541dab0419750bcc521fbdf8585f6e5cb41909df3a472ef7b81ca95"}, + {file = "setuptools-70.1.1.tar.gz", hash = "sha256:937a48c7cdb7a21eb53cd7f9b59e525503aa8abaf3584c730dc5f7a5bec3a650"}, ] [package.extras] docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.1)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test", "mypy (==1.10.0)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.1)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.3.2)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] [[package]] name = "shapely" @@ -4143,64 +4143,64 @@ files = [ [[package]] name = "sqlalchemy" -version = "2.0.30" +version = "2.0.31" description = "Database Abstraction Library" optional = false python-versions = ">=3.7" files = [ - {file = "SQLAlchemy-2.0.30-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3b48154678e76445c7ded1896715ce05319f74b1e73cf82d4f8b59b46e9c0ddc"}, - {file = "SQLAlchemy-2.0.30-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2753743c2afd061bb95a61a51bbb6a1a11ac1c44292fad898f10c9839a7f75b2"}, - {file = "SQLAlchemy-2.0.30-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a7bfc726d167f425d4c16269a9a10fe8630ff6d14b683d588044dcef2d0f6be7"}, - {file = "SQLAlchemy-2.0.30-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c4f61ada6979223013d9ab83a3ed003ded6959eae37d0d685db2c147e9143797"}, - {file = "SQLAlchemy-2.0.30-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:3a365eda439b7a00732638f11072907c1bc8e351c7665e7e5da91b169af794af"}, - {file = "SQLAlchemy-2.0.30-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bba002a9447b291548e8d66fd8c96a6a7ed4f2def0bb155f4f0a1309fd2735d5"}, - {file = "SQLAlchemy-2.0.30-cp310-cp310-win32.whl", hash = "sha256:0138c5c16be3600923fa2169532205d18891b28afa817cb49b50e08f62198bb8"}, - {file = "SQLAlchemy-2.0.30-cp310-cp310-win_amd64.whl", hash = "sha256:99650e9f4cf3ad0d409fed3eec4f071fadd032e9a5edc7270cd646a26446feeb"}, - {file = "SQLAlchemy-2.0.30-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:955991a09f0992c68a499791a753523f50f71a6885531568404fa0f231832aa0"}, - {file = "SQLAlchemy-2.0.30-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f69e4c756ee2686767eb80f94c0125c8b0a0b87ede03eacc5c8ae3b54b99dc46"}, - {file = "SQLAlchemy-2.0.30-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69c9db1ce00e59e8dd09d7bae852a9add716efdc070a3e2068377e6ff0d6fdaa"}, - {file = "SQLAlchemy-2.0.30-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a1429a4b0f709f19ff3b0cf13675b2b9bfa8a7e79990003207a011c0db880a13"}, - {file = "SQLAlchemy-2.0.30-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:efedba7e13aa9a6c8407c48facfdfa108a5a4128e35f4c68f20c3407e4376aa9"}, - {file = "SQLAlchemy-2.0.30-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:16863e2b132b761891d6c49f0a0f70030e0bcac4fd208117f6b7e053e68668d0"}, - {file = "SQLAlchemy-2.0.30-cp311-cp311-win32.whl", hash = "sha256:2ecabd9ccaa6e914e3dbb2aa46b76dede7eadc8cbf1b8083c94d936bcd5ffb49"}, - {file = "SQLAlchemy-2.0.30-cp311-cp311-win_amd64.whl", hash = "sha256:0b3f4c438e37d22b83e640f825ef0f37b95db9aa2d68203f2c9549375d0b2260"}, - {file = "SQLAlchemy-2.0.30-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5a79d65395ac5e6b0c2890935bad892eabb911c4aa8e8015067ddb37eea3d56c"}, - {file = "SQLAlchemy-2.0.30-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9a5baf9267b752390252889f0c802ea13b52dfee5e369527da229189b8bd592e"}, - {file = "SQLAlchemy-2.0.30-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3cb5a646930c5123f8461f6468901573f334c2c63c795b9af350063a736d0134"}, - {file = "SQLAlchemy-2.0.30-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:296230899df0b77dec4eb799bcea6fbe39a43707ce7bb166519c97b583cfcab3"}, - {file = "SQLAlchemy-2.0.30-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c62d401223f468eb4da32627bffc0c78ed516b03bb8a34a58be54d618b74d472"}, - {file = "SQLAlchemy-2.0.30-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3b69e934f0f2b677ec111b4d83f92dc1a3210a779f69bf905273192cf4ed433e"}, - {file = "SQLAlchemy-2.0.30-cp312-cp312-win32.whl", hash = "sha256:77d2edb1f54aff37e3318f611637171e8ec71472f1fdc7348b41dcb226f93d90"}, - {file = "SQLAlchemy-2.0.30-cp312-cp312-win_amd64.whl", hash = "sha256:b6c7ec2b1f4969fc19b65b7059ed00497e25f54069407a8701091beb69e591a5"}, - {file = "SQLAlchemy-2.0.30-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5a8e3b0a7e09e94be7510d1661339d6b52daf202ed2f5b1f9f48ea34ee6f2d57"}, - {file = "SQLAlchemy-2.0.30-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b60203c63e8f984df92035610c5fb76d941254cf5d19751faab7d33b21e5ddc0"}, - {file = "SQLAlchemy-2.0.30-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f1dc3eabd8c0232ee8387fbe03e0a62220a6f089e278b1f0aaf5e2d6210741ad"}, - {file = "SQLAlchemy-2.0.30-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:40ad017c672c00b9b663fcfcd5f0864a0a97828e2ee7ab0c140dc84058d194cf"}, - {file = "SQLAlchemy-2.0.30-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:e42203d8d20dc704604862977b1470a122e4892791fe3ed165f041e4bf447a1b"}, - {file = "SQLAlchemy-2.0.30-cp37-cp37m-win32.whl", hash = "sha256:2a4f4da89c74435f2bc61878cd08f3646b699e7d2eba97144030d1be44e27584"}, - {file = "SQLAlchemy-2.0.30-cp37-cp37m-win_amd64.whl", hash = "sha256:b6bf767d14b77f6a18b6982cbbf29d71bede087edae495d11ab358280f304d8e"}, - {file = "SQLAlchemy-2.0.30-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:bc0c53579650a891f9b83fa3cecd4e00218e071d0ba00c4890f5be0c34887ed3"}, - {file = "SQLAlchemy-2.0.30-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:311710f9a2ee235f1403537b10c7687214bb1f2b9ebb52702c5aa4a77f0b3af7"}, - {file = "SQLAlchemy-2.0.30-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:408f8b0e2c04677e9c93f40eef3ab22f550fecb3011b187f66a096395ff3d9fd"}, - {file = "SQLAlchemy-2.0.30-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37a4b4fb0dd4d2669070fb05b8b8824afd0af57587393015baee1cf9890242d9"}, - {file = "SQLAlchemy-2.0.30-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:a943d297126c9230719c27fcbbeab57ecd5d15b0bd6bfd26e91bfcfe64220621"}, - {file = "SQLAlchemy-2.0.30-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:0a089e218654e740a41388893e090d2e2c22c29028c9d1353feb38638820bbeb"}, - {file = "SQLAlchemy-2.0.30-cp38-cp38-win32.whl", hash = "sha256:fa561138a64f949f3e889eb9ab8c58e1504ab351d6cf55259dc4c248eaa19da6"}, - {file = "SQLAlchemy-2.0.30-cp38-cp38-win_amd64.whl", hash = "sha256:7d74336c65705b986d12a7e337ba27ab2b9d819993851b140efdf029248e818e"}, - {file = "SQLAlchemy-2.0.30-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ae8c62fe2480dd61c532ccafdbce9b29dacc126fe8be0d9a927ca3e699b9491a"}, - {file = "SQLAlchemy-2.0.30-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2383146973a15435e4717f94c7509982770e3e54974c71f76500a0136f22810b"}, - {file = "SQLAlchemy-2.0.30-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8409de825f2c3b62ab15788635ccaec0c881c3f12a8af2b12ae4910a0a9aeef6"}, - {file = "SQLAlchemy-2.0.30-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0094c5dc698a5f78d3d1539853e8ecec02516b62b8223c970c86d44e7a80f6c7"}, - {file = "SQLAlchemy-2.0.30-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:edc16a50f5e1b7a06a2dcc1f2205b0b961074c123ed17ebda726f376a5ab0953"}, - {file = "SQLAlchemy-2.0.30-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:f7703c2010355dd28f53deb644a05fc30f796bd8598b43f0ba678878780b6e4c"}, - {file = "SQLAlchemy-2.0.30-cp39-cp39-win32.whl", hash = "sha256:1f9a727312ff6ad5248a4367358e2cf7e625e98b1028b1d7ab7b806b7d757513"}, - {file = "SQLAlchemy-2.0.30-cp39-cp39-win_amd64.whl", hash = "sha256:a0ef36b28534f2a5771191be6edb44cc2673c7b2edf6deac6562400288664221"}, - {file = "SQLAlchemy-2.0.30-py3-none-any.whl", hash = "sha256:7108d569d3990c71e26a42f60474b4c02c8586c4681af5fd67e51a044fdea86a"}, - {file = "SQLAlchemy-2.0.30.tar.gz", hash = "sha256:2b1708916730f4830bc69d6f49d37f7698b5bd7530aca7f04f785f8849e95255"}, + {file = "SQLAlchemy-2.0.31-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f2a213c1b699d3f5768a7272de720387ae0122f1becf0901ed6eaa1abd1baf6c"}, + {file = "SQLAlchemy-2.0.31-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9fea3d0884e82d1e33226935dac990b967bef21315cbcc894605db3441347443"}, + {file = "SQLAlchemy-2.0.31-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3ad7f221d8a69d32d197e5968d798217a4feebe30144986af71ada8c548e9fa"}, + {file = "SQLAlchemy-2.0.31-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f2bee229715b6366f86a95d497c347c22ddffa2c7c96143b59a2aa5cc9eebbc"}, + {file = "SQLAlchemy-2.0.31-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:cd5b94d4819c0c89280b7c6109c7b788a576084bf0a480ae17c227b0bc41e109"}, + {file = "SQLAlchemy-2.0.31-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:750900a471d39a7eeba57580b11983030517a1f512c2cb287d5ad0fcf3aebd58"}, + {file = "SQLAlchemy-2.0.31-cp310-cp310-win32.whl", hash = "sha256:7bd112be780928c7f493c1a192cd8c5fc2a2a7b52b790bc5a84203fb4381c6be"}, + {file = "SQLAlchemy-2.0.31-cp310-cp310-win_amd64.whl", hash = "sha256:5a48ac4d359f058474fadc2115f78a5cdac9988d4f99eae44917f36aa1476327"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f68470edd70c3ac3b6cd5c2a22a8daf18415203ca1b036aaeb9b0fb6f54e8298"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2e2c38c2a4c5c634fe6c3c58a789712719fa1bf9b9d6ff5ebfce9a9e5b89c1ca"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bd15026f77420eb2b324dcb93551ad9c5f22fab2c150c286ef1dc1160f110203"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2196208432deebdfe3b22185d46b08f00ac9d7b01284e168c212919891289396"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:352b2770097f41bff6029b280c0e03b217c2dcaddc40726f8f53ed58d8a85da4"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:56d51ae825d20d604583f82c9527d285e9e6d14f9a5516463d9705dab20c3740"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-win32.whl", hash = "sha256:6e2622844551945db81c26a02f27d94145b561f9d4b0c39ce7bfd2fda5776dac"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-win_amd64.whl", hash = "sha256:ccaf1b0c90435b6e430f5dd30a5aede4764942a695552eb3a4ab74ed63c5b8d3"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3b74570d99126992d4b0f91fb87c586a574a5872651185de8297c6f90055ae42"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6f77c4f042ad493cb8595e2f503c7a4fe44cd7bd59c7582fd6d78d7e7b8ec52c"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cd1591329333daf94467e699e11015d9c944f44c94d2091f4ac493ced0119449"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:74afabeeff415e35525bf7a4ecdab015f00e06456166a2eba7590e49f8db940e"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b9c01990d9015df2c6f818aa8f4297d42ee71c9502026bb074e713d496e26b67"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:66f63278db425838b3c2b1c596654b31939427016ba030e951b292e32b99553e"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-win32.whl", hash = "sha256:0b0f658414ee4e4b8cbcd4a9bb0fd743c5eeb81fc858ca517217a8013d282c96"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-win_amd64.whl", hash = "sha256:fa4b1af3e619b5b0b435e333f3967612db06351217c58bfb50cee5f003db2a5a"}, + {file = "SQLAlchemy-2.0.31-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:f43e93057cf52a227eda401251c72b6fbe4756f35fa6bfebb5d73b86881e59b0"}, + {file = "SQLAlchemy-2.0.31-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d337bf94052856d1b330d5fcad44582a30c532a2463776e1651bd3294ee7e58b"}, + {file = "SQLAlchemy-2.0.31-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c06fb43a51ccdff3b4006aafee9fcf15f63f23c580675f7734245ceb6b6a9e05"}, + {file = "SQLAlchemy-2.0.31-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:b6e22630e89f0e8c12332b2b4c282cb01cf4da0d26795b7eae16702a608e7ca1"}, + {file = "SQLAlchemy-2.0.31-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:79a40771363c5e9f3a77f0e28b3302801db08040928146e6808b5b7a40749c88"}, + {file = "SQLAlchemy-2.0.31-cp37-cp37m-win32.whl", hash = "sha256:501ff052229cb79dd4c49c402f6cb03b5a40ae4771efc8bb2bfac9f6c3d3508f"}, + {file = "SQLAlchemy-2.0.31-cp37-cp37m-win_amd64.whl", hash = "sha256:597fec37c382a5442ffd471f66ce12d07d91b281fd474289356b1a0041bdf31d"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:dc6d69f8829712a4fd799d2ac8d79bdeff651c2301b081fd5d3fe697bd5b4ab9"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:23b9fbb2f5dd9e630db70fbe47d963c7779e9c81830869bd7d137c2dc1ad05fb"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2a21c97efcbb9f255d5c12a96ae14da873233597dfd00a3a0c4ce5b3e5e79704"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:26a6a9837589c42b16693cf7bf836f5d42218f44d198f9343dd71d3164ceeeac"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:dc251477eae03c20fae8db9c1c23ea2ebc47331bcd73927cdcaecd02af98d3c3"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:2fd17e3bb8058359fa61248c52c7b09a97cf3c820e54207a50af529876451808"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-win32.whl", hash = "sha256:c76c81c52e1e08f12f4b6a07af2b96b9b15ea67ccdd40ae17019f1c373faa227"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-win_amd64.whl", hash = "sha256:4b600e9a212ed59355813becbcf282cfda5c93678e15c25a0ef896b354423238"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b6cf796d9fcc9b37011d3f9936189b3c8074a02a4ed0c0fbbc126772c31a6d4"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:78fe11dbe37d92667c2c6e74379f75746dc947ee505555a0197cfba9a6d4f1a4"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2fc47dc6185a83c8100b37acda27658fe4dbd33b7d5e7324111f6521008ab4fe"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a41514c1a779e2aa9a19f67aaadeb5cbddf0b2b508843fcd7bafdf4c6864005"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:afb6dde6c11ea4525318e279cd93c8734b795ac8bb5dda0eedd9ebaca7fa23f1"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:3f9faef422cfbb8fd53716cd14ba95e2ef655400235c3dfad1b5f467ba179c8c"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-win32.whl", hash = "sha256:fc6b14e8602f59c6ba893980bea96571dd0ed83d8ebb9c4479d9ed5425d562e9"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-win_amd64.whl", hash = "sha256:3cb8a66b167b033ec72c3812ffc8441d4e9f5f78f5e31e54dcd4c90a4ca5bebc"}, + {file = "SQLAlchemy-2.0.31-py3-none-any.whl", hash = "sha256:69f3e3c08867a8e4856e92d7afb618b95cdee18e0bc1647b77599722c9a28911"}, + {file = "SQLAlchemy-2.0.31.tar.gz", hash = "sha256:b607489dd4a54de56984a0c7656247504bd5523d9d0ba799aef59d4add009484"}, ] [package.dependencies] -greenlet = {version = "!=0.4.17", markers = "platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\""} +greenlet = {version = "!=0.4.17", markers = "python_version < \"3.13\" and (platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\")"} typing-extensions = ">=4.6.0" [package.extras] @@ -4748,4 +4748,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.12.2" -content-hash = "122fe6459adad8b8105d12739e4db159ae39fc32e969d02c7c24436ed3fb85b1" +content-hash = "b1b4bfbfdc1f5cc9ae9d090f35b235a62e9dbabc683a5b5a1d0d414605219b48" diff --git a/pyproject.toml b/pyproject.toml index ff5d6a959..2ea004520 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,8 +11,8 @@ python = "^3.12.2" alembic = "==1.13.1" amqp = "==5.2.0" beautifulsoup4 = "==4.12.3" -boto3 = "^1.34.128" -botocore = "^1.34.128" +boto3 = "^1.34.131" +botocore = "^1.34.133" cachetools = "==5.3.3" celery = {version = "==5.4.0", extras = ["redis"]} certifi = ">=2022.12.7" @@ -46,7 +46,7 @@ poetry-dotenv-plugin = "==0.2.0" psycopg2-binary = "==2.9.9" pyjwt = "==2.8.0" python-dotenv = "==1.0.1" -sqlalchemy = "==2.0.30" +sqlalchemy = "==2.0.31" werkzeug = "^3.0.3" faker = "^25.8.0" async-timeout = "^4.0.3" @@ -102,7 +102,7 @@ pytest-cov = "^5.0.0" pytest-xdist = "^3.5.0" radon = "^6.0.1" requests-mock = "^1.11.0" -setuptools = "^70.0.0" +setuptools = "^70.1.1" sqlalchemy-utils = "^0.41.2" vulture = "^2.10" detect-secrets = "^1.5.0" diff --git a/terraform/bootstrap/main.tf b/terraform/bootstrap/main.tf index 14ab9a81d..a518530f1 100644 --- a/terraform/bootstrap/main.tf +++ b/terraform/bootstrap/main.tf @@ -3,7 +3,7 @@ locals { } module "s3" { - source = "github.com/18f/terraform-cloudgov//s3?ref=v0.9.1" + source = "github.com/GSA-TTS/terraform-cloudgov//s3?ref=v1.0.0" cf_org_name = "gsa-tts-benefits-studio" cf_space_name = "notify-management" @@ -13,4 +13,8 @@ module "s3" { resource "cloudfoundry_service_key" "bucket_creds" { name = "${local.s3_service_name}-access" service_instance = module.s3.bucket_id + + lifecycle { + prevent_destroy = true + } } diff --git a/terraform/demo/main.tf b/terraform/demo/main.tf index a791082b3..def3903a0 100644 --- a/terraform/demo/main.tf +++ b/terraform/demo/main.tf @@ -1,12 +1,10 @@ locals { - cf_org_name = "gsa-tts-benefits-studio" - cf_space_name = "notify-demo" - env = "demo" - app_name = "notify-api" - recursive_delete = false # deprecated, still used in shared modules + cf_org_name = "gsa-tts-benefits-studio" + cf_space_name = "notify-demo" + env = "demo" + app_name = "notify-api" } - resource "null_resource" "prevent_destroy" { lifecycle { diff --git a/terraform/production/main.tf b/terraform/production/main.tf index 5a2c520b1..feadd4833 100644 --- a/terraform/production/main.tf +++ b/terraform/production/main.tf @@ -1,38 +1,55 @@ locals { - cf_org_name = "gsa-tts-benefits-studio" - cf_space_name = "notify-production" - env = "production" - app_name = "notify-api" - recursive_delete = false + cf_org_name = "gsa-tts-benefits-studio" + cf_space_name = "notify-production" + env = "production" + app_name = "notify-api" +} + +resource "null_resource" "prevent_destroy" { + + lifecycle { + prevent_destroy = true + } } module "database" { - source = "github.com/18f/terraform-cloudgov//database?ref=v0.7.1" + source = "github.com/GSA-TTS/terraform-cloudgov//database?ref=v1.0.0" - cf_org_name = local.cf_org_name - cf_space_name = local.cf_space_name - name = "${local.app_name}-rds-${local.env}" - recursive_delete = local.recursive_delete - rds_plan_name = "small-psql-redundant" + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + name = "${local.app_name}-rds-${local.env}" + rds_plan_name = "small-psql-redundant" } -module "redis" { - source = "github.com/18f/terraform-cloudgov//redis?ref=v0.7.1" +module "redis" { # default v6.2; delete after v7.0 resource is bound + source = "github.com/GSA-TTS/terraform-cloudgov//redis?ref=v1.0.0" - cf_org_name = local.cf_org_name - cf_space_name = local.cf_space_name - name = "${local.app_name}-redis-${local.env}" - recursive_delete = local.recursive_delete - redis_plan_name = "redis-3node-large" + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + name = "${local.app_name}-redis-${local.env}" + redis_plan_name = "redis-3node-large" +} + +module "redis-v70" { + source = "github.com/GSA-TTS/terraform-cloudgov//redis?ref=v1.0.0" + + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + name = "${local.app_name}-redis-v70-${local.env}" + redis_plan_name = "redis-dev" + json_params = jsonencode( + { + "engineVersion" : "7.0", + } + ) } module "csv_upload_bucket" { - source = "github.com/18f/terraform-cloudgov//s3?ref=v0.7.1" + source = "github.com/GSA-TTS/terraform-cloudgov//s3?ref=v1.0.0" - cf_org_name = local.cf_org_name - cf_space_name = local.cf_space_name - recursive_delete = local.recursive_delete - name = "${local.app_name}-csv-upload-bucket-${local.env}" + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + name = "${local.app_name}-csv-upload-bucket-${local.env}" } module "egress-space" { @@ -51,7 +68,6 @@ module "ses_email" { cf_org_name = local.cf_org_name cf_space_name = local.cf_space_name name = "${local.app_name}-ses-${local.env}" - recursive_delete = local.recursive_delete aws_region = "us-gov-west-1" email_domain = "notify.gov" mail_from_subdomain = "mail" @@ -64,7 +80,6 @@ module "sns_sms" { cf_org_name = local.cf_org_name cf_space_name = local.cf_space_name name = "${local.app_name}-sns-${local.env}" - recursive_delete = local.recursive_delete aws_region = "us-gov-west-1" monthly_spend_limit = 1000 } diff --git a/terraform/sandbox/main.tf b/terraform/sandbox/main.tf index 2a9fa87aa..380131370 100644 --- a/terraform/sandbox/main.tf +++ b/terraform/sandbox/main.tf @@ -1,9 +1,8 @@ locals { - cf_org_name = "gsa-tts-benefits-studio" - cf_space_name = "notify-sandbox" - env = "sandbox" - app_name = "notify-api" - recursive_delete = true # deprecated, still used in shared modules + cf_org_name = "gsa-tts-benefits-studio" + cf_space_name = "notify-sandbox" + env = "sandbox" + app_name = "notify-api" } resource "null_resource" "prevent_destroy" { @@ -22,15 +21,6 @@ module "database" { rds_plan_name = "micro-psql" } -module "redis" { # default v6.2; delete after v7.0 resource is bound - source = "github.com/GSA-TTS/terraform-cloudgov//redis?ref=v1.0.0" - - cf_org_name = local.cf_org_name - cf_space_name = local.cf_space_name - name = "${local.app_name}-redis-${local.env}" - redis_plan_name = "redis-dev" -} - module "redis-v70" { source = "github.com/GSA-TTS/terraform-cloudgov//redis?ref=v1.0.0" @@ -38,7 +28,7 @@ module "redis-v70" { cf_space_name = local.cf_space_name name = "${local.app_name}-redis-v70-${local.env}" redis_plan_name = "redis-dev" - json_params = jsonencode( + json_params = jsonencode( { "engineVersion" : "7.0", } @@ -71,7 +61,6 @@ module "ses_email" { cf_org_name = local.cf_org_name cf_space_name = local.cf_space_name name = "${local.app_name}-ses-${local.env}" - recursive_delete = local.recursive_delete aws_region = "us-west-2" email_receipt_error = "notify-support@gsa.gov" } @@ -82,7 +71,6 @@ module "sns_sms" { cf_org_name = local.cf_org_name cf_space_name = local.cf_space_name name = "${local.app_name}-sns-${local.env}" - recursive_delete = local.recursive_delete aws_region = "us-east-2" monthly_spend_limit = 1 } diff --git a/terraform/shared/ses/main.tf b/terraform/shared/ses/main.tf index a29a8ce10..4c1bb54b9 100644 --- a/terraform/shared/ses/main.tf +++ b/terraform/shared/ses/main.tf @@ -16,10 +16,9 @@ data "cloudfoundry_service" "ses" { } resource "cloudfoundry_service_instance" "ses" { - name = var.name - space = data.cloudfoundry_space.space.id - service_plan = data.cloudfoundry_service.ses.service_plans["base"] - recursive_delete = var.recursive_delete + name = var.name + space = data.cloudfoundry_space.space.id + service_plan = data.cloudfoundry_service.ses.service_plans["base"] json_params = jsonencode({ region = var.aws_region domain = var.email_domain diff --git a/terraform/shared/ses/variables.tf b/terraform/shared/ses/variables.tf index 74e852cf6..a92261656 100644 --- a/terraform/shared/ses/variables.tf +++ b/terraform/shared/ses/variables.tf @@ -13,12 +13,6 @@ variable "name" { description = "name of the service instance" } -variable "recursive_delete" { - type = bool - description = "when true, deletes service bindings attached to the resource (not recommended for production)" - default = false -} - variable "aws_region" { type = string description = "AWS region the SES instance is in" diff --git a/terraform/shared/sns/main.tf b/terraform/shared/sns/main.tf index a23c4e872..aa0079f92 100644 --- a/terraform/shared/sns/main.tf +++ b/terraform/shared/sns/main.tf @@ -16,10 +16,9 @@ data "cloudfoundry_service" "sns" { } resource "cloudfoundry_service_instance" "sns" { - name = var.name - space = data.cloudfoundry_space.space.id - service_plan = data.cloudfoundry_service.sns.service_plans["base"] - recursive_delete = var.recursive_delete + name = var.name + space = data.cloudfoundry_space.space.id + service_plan = data.cloudfoundry_service.sns.service_plans["base"] json_params = jsonencode({ region = var.aws_region monthly_spend_limit = var.monthly_spend_limit diff --git a/terraform/shared/sns/variables.tf b/terraform/shared/sns/variables.tf index 611050337..acf7c5010 100644 --- a/terraform/shared/sns/variables.tf +++ b/terraform/shared/sns/variables.tf @@ -13,12 +13,6 @@ variable "name" { description = "name of the service instance" } -variable "recursive_delete" { - type = bool - description = "when true, deletes service bindings attached to the resource (not recommended for production)" - default = false -} - variable "aws_region" { type = string description = "AWS region the SNS settings are set in" diff --git a/terraform/staging/main.tf b/terraform/staging/main.tf index c46e0d3fa..4fdbf9e38 100644 --- a/terraform/staging/main.tf +++ b/terraform/staging/main.tf @@ -1,38 +1,46 @@ locals { - cf_org_name = "gsa-tts-benefits-studio" - cf_space_name = "notify-staging" - env = "staging" - app_name = "notify-api" - recursive_delete = true + cf_org_name = "gsa-tts-benefits-studio" + cf_space_name = "notify-staging" + env = "staging" + app_name = "notify-api" +} + +resource "null_resource" "prevent_destroy" { + + lifecycle { + prevent_destroy = false # destroying staging is allowed + } } module "database" { - source = "github.com/18f/terraform-cloudgov//database?ref=v0.7.1" + source = "github.com/GSA-TTS/terraform-cloudgov//database?ref=v1.0.0" - cf_org_name = local.cf_org_name - cf_space_name = local.cf_space_name - name = "${local.app_name}-rds-${local.env}" - recursive_delete = local.recursive_delete - rds_plan_name = "micro-psql" + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + name = "${local.app_name}-rds-${local.env}" + rds_plan_name = "micro-psql" } -module "redis" { - source = "github.com/18f/terraform-cloudgov//redis?ref=v0.7.1" +module "redis-v70" { + source = "github.com/GSA-TTS/terraform-cloudgov//redis?ref=v1.0.0" - cf_org_name = local.cf_org_name - cf_space_name = local.cf_space_name - name = "${local.app_name}-redis-${local.env}" - recursive_delete = local.recursive_delete - redis_plan_name = "redis-dev" + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + name = "${local.app_name}-redis-v70-${local.env}" + redis_plan_name = "redis-dev" + json_params = jsonencode( + { + "engineVersion" : "7.0", + } + ) } module "csv_upload_bucket" { - source = "github.com/18f/terraform-cloudgov//s3?ref=v0.7.1" + source = "github.com/GSA-TTS/terraform-cloudgov//s3?ref=v1.0.0" - cf_org_name = local.cf_org_name - cf_space_name = local.cf_space_name - recursive_delete = local.recursive_delete - name = "${local.app_name}-csv-upload-bucket-${local.env}" + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + name = "${local.app_name}-csv-upload-bucket-${local.env}" } module "egress-space" { @@ -53,7 +61,6 @@ module "ses_email" { cf_org_name = local.cf_org_name cf_space_name = local.cf_space_name name = "${local.app_name}-ses-${local.env}" - recursive_delete = local.recursive_delete aws_region = "us-west-2" mail_from_subdomain = "mail" email_receipt_error = "notify-support@gsa.gov" @@ -65,7 +72,6 @@ module "sns_sms" { cf_org_name = local.cf_org_name cf_space_name = local.cf_space_name name = "${local.app_name}-sns-${local.env}" - recursive_delete = local.recursive_delete aws_region = "us-west-2" monthly_spend_limit = 25 } diff --git a/tests/app/service_invite/test_service_invite_rest.py b/tests/app/service_invite/test_service_invite_rest.py index 0f60dc3e3..07d0b4c23 100644 --- a/tests/app/service_invite/test_service_invite_rest.py +++ b/tests/app/service_invite/test_service_invite_rest.py @@ -31,8 +31,8 @@ def test_create_invited_user( extra_args, expected_start_of_invite_url, ): - mocker.patch("app.service_invite.rest.redis_store.raw_set") - mocker.patch("app.service_invite.rest.redis_store.raw_get") + mocker.patch("app.service_invite.rest.redis_store.set") + mocker.patch("app.service_invite.rest.redis_store.get") mocked = mocker.patch("app.celery.provider_tasks.deliver_email.apply_async") email_address = "invited_user@service.gov.uk" @@ -96,8 +96,8 @@ def test_create_invited_user_without_auth_type( admin_request, sample_service, mocker, invitation_email_template ): - mocker.patch("app.service_invite.rest.redis_store.raw_set") - mocker.patch("app.service_invite.rest.redis_store.raw_get") + mocker.patch("app.service_invite.rest.redis_store.set") + mocker.patch("app.service_invite.rest.redis_store.get") mocker.patch("app.celery.provider_tasks.deliver_email.apply_async") email_address = "invited_user@service.gov.uk" invite_from = sample_service.users[0] @@ -220,8 +220,8 @@ def test_resend_expired_invite( mocker, ): - mocker.patch("app.service_invite.rest.redis_store.raw_set") - mocker.patch("app.service_invite.rest.redis_store.raw_get") + mocker.patch("app.service_invite.rest.redis_store.set") + mocker.patch("app.service_invite.rest.redis_store.get") url = f"/service/{sample_expired_user.service_id}/invite/{sample_expired_user.id}/resend" mock_send = mocker.patch("app.service_invite.rest.send_notification_to_queue") mock_persist = mocker.patch("app.service_invite.rest.persist_notification") diff --git a/tests/app/test_utils.py b/tests/app/test_utils.py index 20675aec5..b39d3a2c0 100644 --- a/tests/app/test_utils.py +++ b/tests/app/test_utils.py @@ -13,7 +13,6 @@ from app.utils import ( get_reference_from_personalisation, get_uuid_string_or_none, midnight_n_days_ago, - scrub, ) @@ -95,16 +94,6 @@ def test_get_public_notify_type_text(): ) -def test_scrub(): - result = scrub( - "This is a message with 17775554324, and also 18884449323 and also 17775554324" - ) - assert ( - result - == "This is a message with 1XXXXX54324, and also 1XXXXX49323 and also 1XXXXX54324" - ) - - # This method is used for simulating bulk sends. We use localstack and run on a developer's machine to do the # simulation. Please see docs->bulk_testing.md for instructions. # def test_generate_csv_for_bulk_testing(): diff --git a/tests/app/user/test_rest_verify.py b/tests/app/user/test_rest_verify.py index 81d813039..ff74f6b57 100644 --- a/tests/app/user/test_rest_verify.py +++ b/tests/app/user/test_rest_verify.py @@ -201,10 +201,10 @@ def test_send_user_sms_code(client, sample_user, sms_code_template, mocker): """ notify_service = dao_fetch_service_by_id(current_app.config["NOTIFY_SERVICE_ID"]) - mock_redis_get = mocker.patch("app.user.rest.redis_store.raw_get") + mock_redis_get = mocker.patch("app.user.rest.redis_store.get") mock_redis_get.return_value = "foo" - mocker.patch("app.user.rest.redis_store.raw_set") + mocker.patch("app.user.rest.redis_store.set") auth_header = create_admin_authorization_header() mocked = mocker.patch("app.user.rest.create_secret_code", return_value="11111") mocker.patch("app.celery.provider_tasks.deliver_sms.apply_async") @@ -242,10 +242,10 @@ def test_send_user_code_for_sms_with_optional_to_field( Tests POST endpoint /user//sms-code with optional to field """ - mock_redis_get = mocker.patch("app.user.rest.redis_store.raw_get") + mock_redis_get = mocker.patch("app.user.rest.redis_store.get") mock_redis_get.return_value = "foo" - mocker.patch("app.user.rest.redis_store.raw_set") + mocker.patch("app.user.rest.redis_store.set") to_number = "+14254147755" mocked = mocker.patch("app.user.rest.create_secret_code", return_value="11111") mocker.patch("app.celery.provider_tasks.deliver_sms.apply_async") @@ -469,10 +469,10 @@ def test_send_user_email_code( deliver_email = mocker.patch("app.celery.provider_tasks.deliver_email.apply_async") sample_user.auth_type = auth_type - mock_redis_get = mocker.patch("app.user.rest.redis_store.raw_get") + mock_redis_get = mocker.patch("app.user.rest.redis_store.get") mock_redis_get.return_value = "foo" - mocker.patch("app.user.rest.redis_store.raw_set") + mocker.patch("app.user.rest.redis_store.set") admin_request.post( "user.send_user_2fa_code", @@ -497,10 +497,10 @@ def test_send_user_email_code_with_urlencoded_next_param( ): mocker.patch("app.celery.provider_tasks.deliver_email.apply_async") - mock_redis_get = mocker.patch("app.celery.scheduled_tasks.redis_store.raw_get") + mock_redis_get = mocker.patch("app.celery.scheduled_tasks.redis_store.get") mock_redis_get.return_value = "foo" - mocker.patch("app.celery.scheduled_tasks.redis_store.raw_set") + mocker.patch("app.celery.scheduled_tasks.redis_store.set") data = {"to": None, "next": "/services"} admin_request.post( @@ -581,10 +581,10 @@ def test_user_verify_email_code_fails_if_code_already_used( def test_send_user_2fa_code_sends_from_number_for_international_numbers( client, sample_user, mocker, sms_code_template ): - mock_redis_get = mocker.patch("app.user.rest.redis_store.raw_get") + mock_redis_get = mocker.patch("app.user.rest.redis_store.get") mock_redis_get.return_value = "foo" - mocker.patch("app.user.rest.redis_store.raw_set") + mocker.patch("app.user.rest.redis_store.set") sample_user.mobile_number = "+601117224412" auth_header = create_admin_authorization_header() diff --git a/tests/notifications_utils/clients/redis/test_redis_client.py b/tests/notifications_utils/clients/redis/test_redis_client.py index c9eb63240..52b227128 100644 --- a/tests/notifications_utils/clients/redis/test_redis_client.py +++ b/tests/notifications_utils/clients/redis/test_redis_client.py @@ -1,5 +1,5 @@ import uuid -from unittest.mock import Mock, call +from unittest.mock import Mock import pytest from freezegun import freeze_time @@ -59,30 +59,6 @@ def failing_redis_client(mocked_redis_client, delete_mock): return mocked_redis_client -def test_should_not_raise_exception_if_raise_set_to_false( - app, caplog, failing_redis_client, mocker -): - mock_logger = mocker.patch("flask.Flask.logger") - - assert failing_redis_client.get("get_key") is None - assert failing_redis_client.set("set_key", "set_value") is None - assert failing_redis_client.incr("incr_key") is None - assert failing_redis_client.exceeded_rate_limit("rate_limit_key", 100, 100) is False - assert failing_redis_client.delete("delete_key") is None - assert failing_redis_client.delete("a", "b", "c") is None - assert failing_redis_client.delete_by_pattern("pattern") == 0 - - assert mock_logger.mock_calls == [ - call.exception("Redis error performing get on get_key"), - call.exception("Redis error performing set on set_key"), - call.exception("Redis error performing incr on incr_key"), - call.exception("Redis error performing rate-limit-pipeline on rate_limit_key"), - call.exception("Redis error performing delete on delete_key"), - call.exception("Redis error performing delete on a, b, c"), - call.exception("Redis error performing delete-by-pattern on pattern"), - ] - - def test_should_raise_exception_if_raise_set_to_true( app, failing_redis_client, diff --git a/tests/notifications_utils/test_logging.py b/tests/notifications_utils/test_logging.py index 1aefb4065..2e6362a9c 100644 --- a/tests/notifications_utils/test_logging.py +++ b/tests/notifications_utils/test_logging.py @@ -49,3 +49,18 @@ def test_base_json_formatter_contains_service_id(): == "message to log" ) assert service_id_filter.filter(record).service_id == "notify-admin" + + +def test_pii_filter(): + record = builtin_logging.LogRecord( + name="log thing", + level="info", + pathname="path", + lineno=123, + msg="phone1: 1555555555, phone2: 1555555554, email1: fake@fake.gov, email2: fake@fake2.fake.gov", + exc_info=None, + args=None, + ) + pii_filter = logging.PIIFilter() + clean_msg = "phone1: 1XXXXXXXXXX, phone2: 1XXXXXXXXXX, email1: XXXXX@XXXXXXX, email2: XXXXX@XXXXXXX" + assert pii_filter.filter(record).msg == clean_msg