From abce798f5b119069479d40f7946d5b5eca9a3159 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Mon, 4 Mar 2024 10:32:19 -0800 Subject: [PATCH 01/20] remove celery from notifications-utils (notify-api-130) --- app/__init__.py | 79 +++++++++++++++++++++++++++++++++++++++++++++++-- poetry.lock | 9 +++--- pyproject.toml | 2 +- 3 files changed, 82 insertions(+), 8 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index 5a364f151..9184c4815 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -3,15 +3,16 @@ import secrets import string import time import uuid +from contextlib import contextmanager from time import monotonic -from celery import current_task +from celery import Celery, Task, current_task from flask import current_app, g, has_request_context, jsonify, make_response, request +from flask.ctx import has_app_context from flask_marshmallow import Marshmallow from flask_migrate import Migrate from flask_sqlalchemy import SQLAlchemy as _SQLAlchemy from notifications_utils import logging, request_helper -from notifications_utils.celery import NotifyCelery from notifications_utils.clients.encryption.encryption_client import Encryption from notifications_utils.clients.redis.redis_client import RedisClient from notifications_utils.clients.zendesk.zendesk_client import ZendeskClient @@ -27,6 +28,25 @@ from app.clients.email.aws_ses_stub import AwsSesStubClient from app.clients.sms.aws_sns import AwsSnsClient +class NotifyCelery(Celery): + def init_app(self, app): + self.task_cls = make_task(app) + + # Configure Celery app with options from the main app config. + self.config_from_object(app.config["CELERY"]) + + def send_task(self, name, args=None, kwargs=None, **other_kwargs): + other_kwargs["headers"] = other_kwargs.get("headers") or {} + + if has_request_context() and hasattr(request, "request_id"): + other_kwargs["headers"]["notify_request_id"] = request.request_id + + elif has_app_context() and "request_id" in g: + other_kwargs["headers"]["notify_request_id"] = g.request_id + + return super().send_task(name, args, kwargs, **other_kwargs) + + class SQLAlchemy(_SQLAlchemy): """We need to subclass SQLAlchemy in order to override create_engine options""" @@ -366,3 +386,58 @@ def setup_sqlalchemy_events(app): @event.listens_for(db.engine, "checkin") def checkin(dbapi_connection, connection_record): # noqa pass + + +def make_task(app): + class NotifyTask(Task): + abstract = True + start = None + + @property + def queue_name(self): + delivery_info = self.request.delivery_info or {} + return delivery_info.get("routing_key", "none") + + @property + def request_id(self): + # Note that each header is a direct attribute of the + # task context (aka "request"). + return self.request.get("notify_request_id") + + @contextmanager + def app_context(self): + with app.app_context(): + # Add 'request_id' to 'g' so that it gets logged. + g.request_id = self.request_id + yield + + def on_success(self, retval, task_id, args, kwargs): # noqa + # enables request id tracing for these logs + with self.app_context(): + elapsed_time = time.monotonic() - self.start + + app.logger.info( + "Celery task {task_name} (queue: {queue_name}) took {time}".format( + task_name=self.name, + queue_name=self.queue_name, + time="{0:.4f}".format(elapsed_time), + ) + ) + + def on_failure(self, exc, task_id, args, kwargs, einfo): # noqa + # enables request id tracing for these logs + with self.app_context(): + app.logger.exception( + "Celery task {task_name} (queue: {queue_name}) failed".format( + task_name=self.name, + queue_name=self.queue_name, + ) + ) + + def __call__(self, *args, **kwargs): + # ensure task has flask context to access config, logger, etc + with self.app_context(): + self.start = time.monotonic() + return super().__call__(*args, **kwargs) + + return NotifyTask diff --git a/poetry.lock b/poetry.lock index 8f8b2f11f..2030fc693 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. [[package]] name = "aiohttp" @@ -2670,8 +2670,8 @@ werkzeug = "^3.0.1" [package.source] type = "git" url = "https://github.com/GSA/notifications-utils.git" -reference = "HEAD" -resolved_reference = "df8ece836cad303c5d161edf485cf9bbdc666688" +reference = "053fe30" +resolved_reference = "053fe304a1bbc1ae6559869b1fd0a2c3f83981ea" [[package]] name = "numpy" @@ -3479,7 +3479,6 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, @@ -4771,4 +4770,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.12" -content-hash = "ccfd855abf7309cf520ea5ca02e77fc9d8ab911819ea9690298fa0549a9d7c42" +content-hash = "1041afd9fe9642567640966a7f7b1129f8f084f169419d8f2c7db9071672eb57" diff --git a/pyproject.toml b/pyproject.toml index 970059f5a..a87fefe96 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,7 @@ marshmallow = "==3.20.2" marshmallow-sqlalchemy = "==0.30.0" newrelic = "*" notifications-python-client = "==9.0.0" -notifications-utils = {git = "https://github.com/GSA/notifications-utils.git"} +notifications-utils = {git = "https://github.com/GSA/notifications-utils.git", rev="053fe30"} oscrypto = "==1.3.0" packaging = "==23.2" poetry-dotenv-plugin = "==0.2.0" From 7c95211649f51122202a872a95951aee24c44e8b Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Fri, 8 Mar 2024 08:44:27 -0800 Subject: [PATCH 02/20] fix login.gov to use user uuid instead of email (notify-admin-1277) --- app/dao/users_dao.py | 26 ++++++++++++++++++++++ app/models.py | 1 + app/user/rest.py | 11 +++++++++ migrations/versions/0411_add_login_uuid.py | 20 +++++++++++++++++ tests/app/dao/test_users_dao.py | 7 ++++++ 5 files changed, 65 insertions(+) create mode 100644 migrations/versions/0411_add_login_uuid.py diff --git a/app/dao/users_dao.py b/app/dao/users_dao.py index 7828c5c23..8180e6f11 100644 --- a/app/dao/users_dao.py +++ b/app/dao/users_dao.py @@ -25,6 +25,32 @@ def create_secret_code(length=6): return "{:0{length}d}".format(random_number, length=length) +def get_login_gov_user(login_uuid, email_address): + """ + We want to check to see if the user is registered with login.gov + If we can find the login.gov uuid in our user table, then they are. + + Also, because we originally keyed off email address we might have a few + older users who registered with login.gov but we don't know what their + login.gov uuids are. Eventually the code that checks by email address + should be removed. + """ + + print(User.query.filter_by(login_uuid=login_uuid).first()) + user = User.query.filter_by(login_uuid=login_uuid).first() + if user: + if user.email_address != email_address: + save_user_attribute(user, {"email_address": email_address}) + return user + # Remove this 1 July 2025, all users should have login.gov uuids by now + user = User.query.filter_by(email_address=email_address).first() + if user: + save_user_attribute(user, {"login_uuid": login_uuid}) + return user + + return None + + def save_user_attribute(usr, update_dict=None): db.session.query(User).filter_by(id=usr.id).update(update_dict or {}) db.session.commit() diff --git a/app/models.py b/app/models.py index 07806365f..953624126 100644 --- a/app/models.py +++ b/app/models.py @@ -109,6 +109,7 @@ class User(db.Model): id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) name = db.Column(db.String, nullable=False, index=True, unique=False) email_address = db.Column(db.String(255), nullable=False, index=True, unique=True) + login_uuid = db.Column(db.Text, nullable=True, index=True, unique=True) created_at = db.Column( db.DateTime, index=False, diff --git a/app/user/rest.py b/app/user/rest.py index 098d23de8..8f5f38833 100644 --- a/app/user/rest.py +++ b/app/user/rest.py @@ -19,6 +19,7 @@ from app.dao.users_dao import ( create_secret_code, create_user_code, dao_archive_user, + get_login_gov_user, get_user_and_accounts, get_user_by_email, get_user_by_id, @@ -528,6 +529,16 @@ def set_permissions(user_id, service_id): return jsonify({}), 204 +@user_blueprint.route("/get-login-gov-user", methods=["POST"]) +def get_user_login_gov_user(): + request_args = request.get_json() + login_uuid = request_args["login_uuid"] + email = request_args["email"] + user = get_login_gov_user(login_uuid, email) + result = user.serialize() + return jsonify(data=result) + + @user_blueprint.route("/email", methods=["POST"]) def fetch_user_by_email(): email = email_data_request_schema.load(request.get_json()) diff --git a/migrations/versions/0411_add_login_uuid.py b/migrations/versions/0411_add_login_uuid.py new file mode 100644 index 000000000..88032bf89 --- /dev/null +++ b/migrations/versions/0411_add_login_uuid.py @@ -0,0 +1,20 @@ +""" + +Revision ID: 0411_add_login_uuid +Revises: 410_enums_for_everything +Create Date: 2023-04-24 11:35:22.873930 + +""" +import sqlalchemy as sa +from alembic import op + +revision = "0411_add_login_uuid" +down_revision = "0410_enums_for_everything" + + +def upgrade(): + op.add_column("users", sa.Column("login_uuid", sa.Text)) + + +def downgrade(): + op.drop_column("users", "login_uuid") diff --git a/tests/app/dao/test_users_dao.py b/tests/app/dao/test_users_dao.py index 57ed65619..e38a395b5 100644 --- a/tests/app/dao/test_users_dao.py +++ b/tests/app/dao/test_users_dao.py @@ -15,6 +15,7 @@ from app.dao.users_dao import ( dao_archive_user, delete_codes_older_created_more_than_a_day_ago, delete_model_user, + get_login_gov_user, get_user_by_email, get_user_by_id, increment_failed_login_count, @@ -110,6 +111,12 @@ def test_get_user_by_email(sample_user): assert sample_user == user_from_db +def test_get_login_gov_user(sample_user): + user_from_db = get_login_gov_user("fake_login_gov_uuid", sample_user.email_address) + assert sample_user.email_address == user_from_db.email_address + assert user_from_db.login_uuid is not None + + def test_get_user_by_email_is_case_insensitive(sample_user): email = sample_user.email_address user_from_db = get_user_by_email(email.upper()) From 6c4c8b075a37c07e34610fcb19d4c018502b7163 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Mon, 11 Mar 2024 10:34:59 -0700 Subject: [PATCH 03/20] fix code coverage reporting --- .github/workflows/checks.yml | 3 ++- Makefile | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 280d2566e..dd0fe3275 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -61,7 +61,8 @@ jobs: NOTIFY_E2E_TEST_HTTP_AUTH_USER: ${{ secrets.NOTIFY_E2E_TEST_HTTP_AUTH_USER }} NOTIFY_E2E_TEST_PASSWORD: ${{ secrets.NOTIFY_E2E_TEST_PASSWORD }} - name: Check coverage threshold - run: poetry run coverage report --fail-under=50 + # TODO get this back up to 95 + run: poetry run coverage report --fail-under=87 validate-new-relic-config: runs-on: ubuntu-latest diff --git a/Makefile b/Makefile index 4273f7b7e..e86cfc1c3 100644 --- a/Makefile +++ b/Makefile @@ -79,7 +79,7 @@ test: ## Run tests and create coverage report poetry run black . poetry run flake8 . poetry run isort --check-only ./app ./tests - poetry run coverage run -m pytest -vv --maxfail=10 + poetry run coverage run --omit=*/notifications_utils/* -m pytest --maxfail=10 poetry run coverage report -m --fail-under=95 poetry run coverage html -d .coverage_cache From 61703471bbad855c972a9e9062a8977cab59db4f Mon Sep 17 00:00:00 2001 From: Carlo Costino Date: Tue, 12 Mar 2024 13:59:08 -0400 Subject: [PATCH 04/20] Fix remaining Terraform for production and demo This changeset adjusts our Terraform for the production and demo environments so that the new delete_recursive_allowed with the Cloud Foundry Cloud Controller is set to false. It also updates the shared modules to all explicitly account for this property so that we can set the flag easily. Signed-off-by: Carlo Costino --- terraform/demo/main.tf | 73 ++++++++++++++------------ terraform/demo/providers.tf | 2 +- terraform/production/main.tf | 75 +++++++++++++++------------ terraform/production/providers.tf | 2 +- terraform/shared/egress_space/main.tf | 2 +- terraform/shared/ses/main.tf | 9 ++++ terraform/shared/ses/variables.tf | 6 +++ terraform/shared/sns/main.tf | 9 ++++ terraform/shared/sns/variables.tf | 6 +++ 9 files changed, 116 insertions(+), 68 deletions(-) diff --git a/terraform/demo/main.tf b/terraform/demo/main.tf index 615f92670..5f3f8525e 100644 --- a/terraform/demo/main.tf +++ b/terraform/demo/main.tf @@ -1,38 +1,46 @@ locals { - cf_org_name = "gsa-tts-benefits-studio" - cf_space_name = "notify-demo" - env = "demo" - app_name = "notify-api" - recursive_delete = false + cf_org_name = "gsa-tts-benefits-studio" + cf_space_name = "notify-demo" + env = "demo" + app_name = "notify-api" + delete_recursive_allowed = false +} + +data "cloudfoundry_space" "demo" { + org_name = local.cf_org_name + name = local.cf_space_name +} + +resource "cloudfoundry_space" "notify-demo" { + delete_recursive_allowed = local.delete_recursive_allowed + name = local.cf_space_name + org = data.cloudfoundry_org.org.id } module "database" { source = "github.com/18f/terraform-cloudgov//database?ref=v0.7.1" - cf_org_name = local.cf_org_name - cf_space_name = local.cf_space_name - name = "${local.app_name}-rds-${local.env}" - recursive_delete = local.recursive_delete - rds_plan_name = "micro-psql" + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + name = "${local.app_name}-rds-${local.env}" + rds_plan_name = "micro-psql" } module "redis" { source = "github.com/18f/terraform-cloudgov//redis?ref=v0.7.1" - cf_org_name = local.cf_org_name - cf_space_name = local.cf_space_name - name = "${local.app_name}-redis-${local.env}" - recursive_delete = local.recursive_delete - redis_plan_name = "redis-dev" + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + name = "${local.app_name}-redis-${local.env}" + redis_plan_name = "redis-dev" } module "csv_upload_bucket" { source = "github.com/18f/terraform-cloudgov//s3?ref=v0.7.1" - cf_org_name = local.cf_org_name - cf_space_name = local.cf_space_name - recursive_delete = local.recursive_delete - name = "${local.app_name}-csv-upload-bucket-${local.env}" + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + name = "${local.app_name}-csv-upload-bucket-${local.env}" } module "egress-space" { @@ -40,6 +48,7 @@ module "egress-space" { cf_org_name = local.cf_org_name cf_restricted_space_name = local.cf_space_name + delete_recursive_allowed = local.delete_recursive_allowed deployers = [ var.cf_user, "steven.reilly@gsa.gov" @@ -49,22 +58,22 @@ module "egress-space" { module "ses_email" { source = "../shared/ses" - cf_org_name = local.cf_org_name - cf_space_name = local.cf_space_name - name = "${local.app_name}-ses-${local.env}" - recursive_delete = local.recursive_delete - aws_region = "us-west-2" - email_domain = "notify.sandbox.10x.gsa.gov" - email_receipt_error = "notify-support@gsa.gov" + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + name = "${local.app_name}-ses-${local.env}" + aws_region = "us-west-2" + email_domain = "notify.sandbox.10x.gsa.gov" + email_receipt_error = "notify-support@gsa.gov" + delete_recursive_allowed = local.delete_recursive_allowed } module "sns_sms" { source = "../shared/sns" - cf_org_name = local.cf_org_name - cf_space_name = local.cf_space_name - name = "${local.app_name}-sns-${local.env}" - recursive_delete = local.recursive_delete - aws_region = "us-east-1" - monthly_spend_limit = 25 + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + name = "${local.app_name}-sns-${local.env}" + aws_region = "us-east-1" + monthly_spend_limit = 25 + delete_recursive_allowed = local.delete_recursive_allowed } diff --git a/terraform/demo/providers.tf b/terraform/demo/providers.tf index f13333d3e..34ba30a62 100644 --- a/terraform/demo/providers.tf +++ b/terraform/demo/providers.tf @@ -3,7 +3,7 @@ terraform { required_providers { cloudfoundry = { source = "cloudfoundry-community/cloudfoundry" - version = "0.53.0" + version = "0.53.1" } } diff --git a/terraform/production/main.tf b/terraform/production/main.tf index 5a2c520b1..c4ca005f5 100644 --- a/terraform/production/main.tf +++ b/terraform/production/main.tf @@ -1,38 +1,46 @@ locals { - cf_org_name = "gsa-tts-benefits-studio" - cf_space_name = "notify-production" - env = "production" - app_name = "notify-api" - recursive_delete = false + cf_org_name = "gsa-tts-benefits-studio" + cf_space_name = "notify-production" + env = "production" + app_name = "notify-api" + delete_recursive_allowed = false +} + +data "cloudfoundry_space" "production" { + org_name = local.cf_org_name + name = local.cf_space_name +} + +resource "cloudfoundry_space" "notify-production" { + delete_recursive_allowed = local.delete_recursive_allowed + name = local.cf_space_name + org = data.cloudfoundry_org.org.id } module "database" { source = "github.com/18f/terraform-cloudgov//database?ref=v0.7.1" - cf_org_name = local.cf_org_name - cf_space_name = local.cf_space_name - name = "${local.app_name}-rds-${local.env}" - recursive_delete = local.recursive_delete - rds_plan_name = "small-psql-redundant" + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + name = "${local.app_name}-rds-${local.env}" + rds_plan_name = "small-psql-redundant" } module "redis" { source = "github.com/18f/terraform-cloudgov//redis?ref=v0.7.1" - cf_org_name = local.cf_org_name - cf_space_name = local.cf_space_name - name = "${local.app_name}-redis-${local.env}" - recursive_delete = local.recursive_delete - redis_plan_name = "redis-3node-large" + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + name = "${local.app_name}-redis-${local.env}" + redis_plan_name = "redis-3node-large" } module "csv_upload_bucket" { source = "github.com/18f/terraform-cloudgov//s3?ref=v0.7.1" - cf_org_name = local.cf_org_name - cf_space_name = local.cf_space_name - recursive_delete = local.recursive_delete - name = "${local.app_name}-csv-upload-bucket-${local.env}" + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + name = "${local.app_name}-csv-upload-bucket-${local.env}" } module "egress-space" { @@ -40,6 +48,7 @@ module "egress-space" { cf_org_name = local.cf_org_name cf_restricted_space_name = local.cf_space_name + delete_recursive_allowed = local.delete_recursive_allowed deployers = [ var.cf_user ] @@ -48,25 +57,25 @@ module "egress-space" { module "ses_email" { source = "../shared/ses" - cf_org_name = local.cf_org_name - cf_space_name = local.cf_space_name - name = "${local.app_name}-ses-${local.env}" - recursive_delete = local.recursive_delete - aws_region = "us-gov-west-1" - email_domain = "notify.gov" - mail_from_subdomain = "mail" - email_receipt_error = "notify-support@gsa.gov" + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + name = "${local.app_name}-ses-${local.env}" + aws_region = "us-gov-west-1" + email_domain = "notify.gov" + mail_from_subdomain = "mail" + email_receipt_error = "notify-support@gsa.gov" + delete_recursive_allowed = local.delete_recursive_allowed } module "sns_sms" { source = "../shared/sns" - cf_org_name = local.cf_org_name - cf_space_name = local.cf_space_name - name = "${local.app_name}-sns-${local.env}" - recursive_delete = local.recursive_delete - aws_region = "us-gov-west-1" - monthly_spend_limit = 1000 + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + name = "${local.app_name}-sns-${local.env}" + aws_region = "us-gov-west-1" + monthly_spend_limit = 1000 + delete_recursive_allowed = local.delete_recursive_allowed } ########################################################################### diff --git a/terraform/production/providers.tf b/terraform/production/providers.tf index 499759f48..b5c45f63e 100644 --- a/terraform/production/providers.tf +++ b/terraform/production/providers.tf @@ -3,7 +3,7 @@ terraform { required_providers { cloudfoundry = { source = "cloudfoundry-community/cloudfoundry" - version = "0.53.0" + version = "0.53.1" } } diff --git a/terraform/shared/egress_space/main.tf b/terraform/shared/egress_space/main.tf index 5d4b53354..066f0ba58 100644 --- a/terraform/shared/egress_space/main.tf +++ b/terraform/shared/egress_space/main.tf @@ -11,7 +11,7 @@ data "cloudfoundry_org" "org" { ### resource "cloudfoundry_space" "public_egress" { - delete_recursive_allowed = false + delete_recursive_allowed = var.delete_recursive_allowed name = "${var.cf_restricted_space_name}-egress" org = data.cloudfoundry_org.org.id } diff --git a/terraform/shared/ses/main.tf b/terraform/shared/ses/main.tf index 4c1bb54b9..1bee9a74a 100644 --- a/terraform/shared/ses/main.tf +++ b/terraform/shared/ses/main.tf @@ -7,6 +7,15 @@ data "cloudfoundry_space" "space" { name = var.cf_space_name } +### +# SES Space +### +resource "cloudfoundry_space" "cf_ses_service_space" { + delete_recursive_allowed = var.delete_recursive_allowed + name = data.cloudfoundry_space.space.name + org = data.cloudfoundry_org.org.id +} + ### # SES instance ### diff --git a/terraform/shared/ses/variables.tf b/terraform/shared/ses/variables.tf index a92261656..35e8cad8c 100644 --- a/terraform/shared/ses/variables.tf +++ b/terraform/shared/ses/variables.tf @@ -34,3 +34,9 @@ variable "mail_from_subdomain" { description = "Subdomain of email_domain to set as the mail-from header" default = "" } + +variable "delete_recursive_allowed" { + type = bool + default = true + description = "Flag for allowing resources to be recursively deleted - not recommended in production environments" +} diff --git a/terraform/shared/sns/main.tf b/terraform/shared/sns/main.tf index aa0079f92..2df89907d 100644 --- a/terraform/shared/sns/main.tf +++ b/terraform/shared/sns/main.tf @@ -7,6 +7,15 @@ data "cloudfoundry_space" "space" { name = var.cf_space_name } +### +# SNS Space +### +resource "cloudfoundry_space" "cf_sns_service_space" { + delete_recursive_allowed = var.delete_recursive_allowed + name = data.cloudfoundry_space.space.name + org = data.cloudfoundry_org.org.id +} + ### # SES instance ### diff --git a/terraform/shared/sns/variables.tf b/terraform/shared/sns/variables.tf index acf7c5010..801e41504 100644 --- a/terraform/shared/sns/variables.tf +++ b/terraform/shared/sns/variables.tf @@ -22,3 +22,9 @@ variable "monthly_spend_limit" { type = number description = "SMS budget limit in USD. Support request must be made before raising above 1" } + +variable "delete_recursive_allowed" { + type = bool + default = true + description = "Flag for allowing resources to be recursively deleted - not recommended in production environments" +} From ef46ddcb37d2ee4394c72a9d4574674226707837 Mon Sep 17 00:00:00 2001 From: Carlo Costino Date: Tue, 12 Mar 2024 14:19:59 -0400 Subject: [PATCH 05/20] Fixed reference to the Cloud Foundry org instead of space Signed-off-by: Carlo Costino --- terraform/shared/ses/main.tf | 5 ++--- terraform/shared/sns/main.tf | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/terraform/shared/ses/main.tf b/terraform/shared/ses/main.tf index 1bee9a74a..174704d01 100644 --- a/terraform/shared/ses/main.tf +++ b/terraform/shared/ses/main.tf @@ -2,9 +2,8 @@ # Target space/org ### -data "cloudfoundry_space" "space" { - org_name = var.cf_org_name - name = var.cf_space_name +data "cloudfoundry_org" "org" { + name = var.cf_org_name } ### diff --git a/terraform/shared/sns/main.tf b/terraform/shared/sns/main.tf index 2df89907d..57d85e562 100644 --- a/terraform/shared/sns/main.tf +++ b/terraform/shared/sns/main.tf @@ -2,9 +2,8 @@ # Target space/org ### -data "cloudfoundry_space" "space" { - org_name = var.cf_org_name - name = var.cf_space_name +data "cloudfoundry_org" "org" { + name = var.cf_org_name } ### From 68fa12340ad4ad236cb7e9bba5072dda16971aa8 Mon Sep 17 00:00:00 2001 From: Carlo Costino Date: Tue, 12 Mar 2024 14:22:58 -0400 Subject: [PATCH 06/20] Add space data back in - we still need it! Signed-off-by: Carlo Costino --- terraform/shared/ses/main.tf | 5 +++++ terraform/shared/sns/main.tf | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/terraform/shared/ses/main.tf b/terraform/shared/ses/main.tf index 174704d01..764cd3999 100644 --- a/terraform/shared/ses/main.tf +++ b/terraform/shared/ses/main.tf @@ -6,6 +6,11 @@ data "cloudfoundry_org" "org" { name = var.cf_org_name } +data "cloudfoundry_space" "space" { + org_name = var.cf_org_name + name = var.cf_space_name +} + ### # SES Space ### diff --git a/terraform/shared/sns/main.tf b/terraform/shared/sns/main.tf index 57d85e562..46319581f 100644 --- a/terraform/shared/sns/main.tf +++ b/terraform/shared/sns/main.tf @@ -6,6 +6,11 @@ data "cloudfoundry_org" "org" { name = var.cf_org_name } +data "cloudfoundry_space" "space" { + org_name = var.cf_org_name + name = var.cf_space_name +} + ### # SNS Space ### From f0e66886d21dc2c41f8c53db4477834ce7432ae1 Mon Sep 17 00:00:00 2001 From: Carlo Costino Date: Tue, 12 Mar 2024 14:26:28 -0400 Subject: [PATCH 07/20] Adding missing variable for egress_space Signed-off-by: Carlo Costino --- terraform/shared/egress_space/variables.tf | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/terraform/shared/egress_space/variables.tf b/terraform/shared/egress_space/variables.tf index 45bcc717d..6325fa7cf 100644 --- a/terraform/shared/egress_space/variables.tf +++ b/terraform/shared/egress_space/variables.tf @@ -3,3 +3,9 @@ variable "cf_restricted_space_name" {} variable "deployers" { type = set(string) } + +variable "delete_recursive_allowed" { + type = bool + default = true + description = "Flag for allowing resources to be recursively deleted - not recommended in production environments" +} From 111135a9b6a93970117f3868e4bcb5813d5b2904 Mon Sep 17 00:00:00 2001 From: Cliff Hill Date: Tue, 12 Mar 2024 15:05:34 -0400 Subject: [PATCH 08/20] Helping the migration succeed! Signed-off-by: Cliff Hill --- migrations/versions/0410_enums_for_everything.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/migrations/versions/0410_enums_for_everything.py b/migrations/versions/0410_enums_for_everything.py index e34a3621a..b6c9042c6 100644 --- a/migrations/versions/0410_enums_for_everything.py +++ b/migrations/versions/0410_enums_for_everything.py @@ -468,6 +468,19 @@ def upgrade(): existing_nullable=False, postgresql_using=enum_using("notification_type", NotificationType), ) + # Clobbering bad data here. These are values we don't use any more, and anything with them is unnecessary. + op.execute(""" + delete from + service_permissions + where + permission in ( + 'letter', + 'letters_as_pdf', + 'upload_letters', + 'international_letters', + 'broadcast' + ); + """) op.alter_column( "service_permissions", "permission", From 15f8be7aee0ffb97033c5b22c881fd45b65f1a72 Mon Sep 17 00:00:00 2001 From: Carlo Costino Date: Wed, 13 Mar 2024 09:51:41 -0400 Subject: [PATCH 09/20] Explicitly add allow_ssh flag and disable for production This will also ensure any drift is picked up by our infrastructure verification checks Signed-off-by: Carlo Costino --- terraform/production/main.tf | 5 +++++ terraform/shared/egress_space/main.tf | 1 + terraform/shared/egress_space/variables.tf | 6 ++++++ terraform/shared/ses/main.tf | 1 + terraform/shared/ses/variables.tf | 6 ++++++ terraform/shared/sns/main.tf | 1 + terraform/shared/sns/variables.tf | 6 ++++++ 7 files changed, 26 insertions(+) diff --git a/terraform/production/main.tf b/terraform/production/main.tf index c4ca005f5..e2c321d37 100644 --- a/terraform/production/main.tf +++ b/terraform/production/main.tf @@ -4,6 +4,7 @@ locals { env = "production" app_name = "notify-api" delete_recursive_allowed = false + allow_ssh = false } data "cloudfoundry_space" "production" { @@ -12,6 +13,7 @@ data "cloudfoundry_space" "production" { } resource "cloudfoundry_space" "notify-production" { + allow_ssh = local.allow_ssh delete_recursive_allowed = local.delete_recursive_allowed name = local.cf_space_name org = data.cloudfoundry_org.org.id @@ -46,6 +48,7 @@ module "csv_upload_bucket" { module "egress-space" { source = "../shared/egress_space" + allow_ssh = local.allow_ssh cf_org_name = local.cf_org_name cf_restricted_space_name = local.cf_space_name delete_recursive_allowed = local.delete_recursive_allowed @@ -57,6 +60,7 @@ module "egress-space" { module "ses_email" { source = "../shared/ses" + allow_ssh = local.allow_ssh cf_org_name = local.cf_org_name cf_space_name = local.cf_space_name name = "${local.app_name}-ses-${local.env}" @@ -70,6 +74,7 @@ module "ses_email" { module "sns_sms" { source = "../shared/sns" + allow_ssh = local.allow_ssh cf_org_name = local.cf_org_name cf_space_name = local.cf_space_name name = "${local.app_name}-sns-${local.env}" diff --git a/terraform/shared/egress_space/main.tf b/terraform/shared/egress_space/main.tf index 066f0ba58..cc91e9c42 100644 --- a/terraform/shared/egress_space/main.tf +++ b/terraform/shared/egress_space/main.tf @@ -11,6 +11,7 @@ data "cloudfoundry_org" "org" { ### resource "cloudfoundry_space" "public_egress" { + allow_ssh = var.allow_ssh delete_recursive_allowed = var.delete_recursive_allowed name = "${var.cf_restricted_space_name}-egress" org = data.cloudfoundry_org.org.id diff --git a/terraform/shared/egress_space/variables.tf b/terraform/shared/egress_space/variables.tf index 6325fa7cf..5bdff893f 100644 --- a/terraform/shared/egress_space/variables.tf +++ b/terraform/shared/egress_space/variables.tf @@ -9,3 +9,9 @@ variable "delete_recursive_allowed" { default = true description = "Flag for allowing resources to be recursively deleted - not recommended in production environments" } + +variable "allow_ssh" { + type = bool + default = true + description = "Flag for allowing SSH access in a space - not recommended in production environments" +} diff --git a/terraform/shared/ses/main.tf b/terraform/shared/ses/main.tf index 764cd3999..0661d1089 100644 --- a/terraform/shared/ses/main.tf +++ b/terraform/shared/ses/main.tf @@ -15,6 +15,7 @@ data "cloudfoundry_space" "space" { # SES Space ### resource "cloudfoundry_space" "cf_ses_service_space" { + allow_ssh = var.allow_ssh delete_recursive_allowed = var.delete_recursive_allowed name = data.cloudfoundry_space.space.name org = data.cloudfoundry_org.org.id diff --git a/terraform/shared/ses/variables.tf b/terraform/shared/ses/variables.tf index 35e8cad8c..e6a1b2b62 100644 --- a/terraform/shared/ses/variables.tf +++ b/terraform/shared/ses/variables.tf @@ -40,3 +40,9 @@ variable "delete_recursive_allowed" { default = true description = "Flag for allowing resources to be recursively deleted - not recommended in production environments" } + +variable "allow_ssh" { + type = bool + default = true + description = "Flag for allowing SSH access in a space - not recommended in production environments" +} diff --git a/terraform/shared/sns/main.tf b/terraform/shared/sns/main.tf index 46319581f..a00171a98 100644 --- a/terraform/shared/sns/main.tf +++ b/terraform/shared/sns/main.tf @@ -15,6 +15,7 @@ data "cloudfoundry_space" "space" { # SNS Space ### resource "cloudfoundry_space" "cf_sns_service_space" { + allow_ssh = var.allow_ssh delete_recursive_allowed = var.delete_recursive_allowed name = data.cloudfoundry_space.space.name org = data.cloudfoundry_org.org.id diff --git a/terraform/shared/sns/variables.tf b/terraform/shared/sns/variables.tf index 801e41504..d75ddeda3 100644 --- a/terraform/shared/sns/variables.tf +++ b/terraform/shared/sns/variables.tf @@ -28,3 +28,9 @@ variable "delete_recursive_allowed" { default = true description = "Flag for allowing resources to be recursively deleted - not recommended in production environments" } + +variable "allow_ssh" { + type = bool + default = true + description = "Flag for allowing SSH access in a space - not recommended in production environments" +} From 1484c2ffa0b9253632fa5f0ae630b06860595360 Mon Sep 17 00:00:00 2001 From: Carlo Costino Date: Wed, 13 Mar 2024 10:05:02 -0400 Subject: [PATCH 10/20] Adjust properties further to only apply to spaces The SES and SNS modules are strictly for services that are instantiated within a space, while the egress_space is creating a separate space. The shift of the recursive delete from being on a service to being at the space level means that the SES and SNS modules do not have to track it at all, it should only be handled at the space level. The same goes for the allow_ssh flag. Signed-off-by: Carlo Costino --- terraform/demo/main.tf | 24 +++++++++++------------- terraform/production/main.tf | 28 ++++++++++++---------------- terraform/shared/ses/main.tf | 10 ---------- terraform/shared/ses/variables.tf | 12 ------------ terraform/shared/sns/main.tf | 10 ---------- terraform/shared/sns/variables.tf | 12 ------------ 6 files changed, 23 insertions(+), 73 deletions(-) diff --git a/terraform/demo/main.tf b/terraform/demo/main.tf index 5f3f8525e..0ac491fc5 100644 --- a/terraform/demo/main.tf +++ b/terraform/demo/main.tf @@ -58,22 +58,20 @@ module "egress-space" { module "ses_email" { source = "../shared/ses" - cf_org_name = local.cf_org_name - cf_space_name = local.cf_space_name - name = "${local.app_name}-ses-${local.env}" - aws_region = "us-west-2" - email_domain = "notify.sandbox.10x.gsa.gov" - email_receipt_error = "notify-support@gsa.gov" - delete_recursive_allowed = local.delete_recursive_allowed + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + name = "${local.app_name}-ses-${local.env}" + aws_region = "us-west-2" + email_domain = "notify.sandbox.10x.gsa.gov" + email_receipt_error = "notify-support@gsa.gov" } module "sns_sms" { source = "../shared/sns" - cf_org_name = local.cf_org_name - cf_space_name = local.cf_space_name - name = "${local.app_name}-sns-${local.env}" - aws_region = "us-east-1" - monthly_spend_limit = 25 - delete_recursive_allowed = local.delete_recursive_allowed + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + name = "${local.app_name}-sns-${local.env}" + aws_region = "us-east-1" + monthly_spend_limit = 25 } diff --git a/terraform/production/main.tf b/terraform/production/main.tf index e2c321d37..f7fc93d68 100644 --- a/terraform/production/main.tf +++ b/terraform/production/main.tf @@ -60,27 +60,23 @@ module "egress-space" { module "ses_email" { source = "../shared/ses" - allow_ssh = local.allow_ssh - cf_org_name = local.cf_org_name - cf_space_name = local.cf_space_name - name = "${local.app_name}-ses-${local.env}" - aws_region = "us-gov-west-1" - email_domain = "notify.gov" - mail_from_subdomain = "mail" - email_receipt_error = "notify-support@gsa.gov" - delete_recursive_allowed = local.delete_recursive_allowed + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + name = "${local.app_name}-ses-${local.env}" + aws_region = "us-gov-west-1" + email_domain = "notify.gov" + mail_from_subdomain = "mail" + email_receipt_error = "notify-support@gsa.gov" } module "sns_sms" { source = "../shared/sns" - allow_ssh = local.allow_ssh - cf_org_name = local.cf_org_name - cf_space_name = local.cf_space_name - name = "${local.app_name}-sns-${local.env}" - aws_region = "us-gov-west-1" - monthly_spend_limit = 1000 - delete_recursive_allowed = local.delete_recursive_allowed + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + name = "${local.app_name}-sns-${local.env}" + aws_region = "us-gov-west-1" + monthly_spend_limit = 1000 } ########################################################################### diff --git a/terraform/shared/ses/main.tf b/terraform/shared/ses/main.tf index 0661d1089..80c40042b 100644 --- a/terraform/shared/ses/main.tf +++ b/terraform/shared/ses/main.tf @@ -11,16 +11,6 @@ data "cloudfoundry_space" "space" { name = var.cf_space_name } -### -# SES Space -### -resource "cloudfoundry_space" "cf_ses_service_space" { - allow_ssh = var.allow_ssh - delete_recursive_allowed = var.delete_recursive_allowed - name = data.cloudfoundry_space.space.name - org = data.cloudfoundry_org.org.id -} - ### # SES instance ### diff --git a/terraform/shared/ses/variables.tf b/terraform/shared/ses/variables.tf index e6a1b2b62..a92261656 100644 --- a/terraform/shared/ses/variables.tf +++ b/terraform/shared/ses/variables.tf @@ -34,15 +34,3 @@ variable "mail_from_subdomain" { description = "Subdomain of email_domain to set as the mail-from header" default = "" } - -variable "delete_recursive_allowed" { - type = bool - default = true - description = "Flag for allowing resources to be recursively deleted - not recommended in production environments" -} - -variable "allow_ssh" { - type = bool - default = true - description = "Flag for allowing SSH access in a space - not recommended in production environments" -} diff --git a/terraform/shared/sns/main.tf b/terraform/shared/sns/main.tf index a00171a98..73cb8a815 100644 --- a/terraform/shared/sns/main.tf +++ b/terraform/shared/sns/main.tf @@ -11,16 +11,6 @@ data "cloudfoundry_space" "space" { name = var.cf_space_name } -### -# SNS Space -### -resource "cloudfoundry_space" "cf_sns_service_space" { - allow_ssh = var.allow_ssh - delete_recursive_allowed = var.delete_recursive_allowed - name = data.cloudfoundry_space.space.name - org = data.cloudfoundry_org.org.id -} - ### # SES instance ### diff --git a/terraform/shared/sns/variables.tf b/terraform/shared/sns/variables.tf index d75ddeda3..acf7c5010 100644 --- a/terraform/shared/sns/variables.tf +++ b/terraform/shared/sns/variables.tf @@ -22,15 +22,3 @@ variable "monthly_spend_limit" { type = number description = "SMS budget limit in USD. Support request must be made before raising above 1" } - -variable "delete_recursive_allowed" { - type = bool - default = true - description = "Flag for allowing resources to be recursively deleted - not recommended in production environments" -} - -variable "allow_ssh" { - type = bool - default = true - description = "Flag for allowing SSH access in a space - not recommended in production environments" -} From 0404348ec0738117be93e3850732da8570513224 Mon Sep 17 00:00:00 2001 From: Carlo Costino Date: Wed, 13 Mar 2024 10:12:54 -0400 Subject: [PATCH 11/20] Removed last bit of extraneous config that is not needed Signed-off-by: Carlo Costino --- terraform/shared/ses/main.tf | 4 ---- terraform/shared/sns/main.tf | 4 ---- 2 files changed, 8 deletions(-) diff --git a/terraform/shared/ses/main.tf b/terraform/shared/ses/main.tf index 80c40042b..4c1bb54b9 100644 --- a/terraform/shared/ses/main.tf +++ b/terraform/shared/ses/main.tf @@ -2,10 +2,6 @@ # Target space/org ### -data "cloudfoundry_org" "org" { - name = var.cf_org_name -} - data "cloudfoundry_space" "space" { org_name = var.cf_org_name name = var.cf_space_name diff --git a/terraform/shared/sns/main.tf b/terraform/shared/sns/main.tf index 73cb8a815..aa0079f92 100644 --- a/terraform/shared/sns/main.tf +++ b/terraform/shared/sns/main.tf @@ -2,10 +2,6 @@ # Target space/org ### -data "cloudfoundry_org" "org" { - name = var.cf_org_name -} - data "cloudfoundry_space" "space" { org_name = var.cf_org_name name = var.cf_space_name From 9d60e6e8b7350f825215caf68a2f0f4b9050b468 Mon Sep 17 00:00:00 2001 From: Carlo Costino Date: Wed, 13 Mar 2024 11:26:01 -0400 Subject: [PATCH 12/20] Fix reference to CF org vs. space This changeset fixes a reference to properly load the Cloud Foundry org for modifying the space. Signed-off-by: Carlo Costino --- terraform/demo/main.tf | 5 ++--- terraform/production/main.tf | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/terraform/demo/main.tf b/terraform/demo/main.tf index 0ac491fc5..e594264c2 100644 --- a/terraform/demo/main.tf +++ b/terraform/demo/main.tf @@ -6,9 +6,8 @@ locals { delete_recursive_allowed = false } -data "cloudfoundry_space" "demo" { - org_name = local.cf_org_name - name = local.cf_space_name +data "cloudfoundry_org" "org" { + name = local.cf_org_name } resource "cloudfoundry_space" "notify-demo" { diff --git a/terraform/production/main.tf b/terraform/production/main.tf index f7fc93d68..ff1daad88 100644 --- a/terraform/production/main.tf +++ b/terraform/production/main.tf @@ -7,9 +7,8 @@ locals { allow_ssh = false } -data "cloudfoundry_space" "production" { - org_name = local.cf_org_name - name = local.cf_space_name +data "cloudfoundry_org" "org" { + name = local.cf_org_name } resource "cloudfoundry_space" "notify-production" { From a8640a65b6f73c89866d75b1918e41ca6c1577c1 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Wed, 13 Mar 2024 10:52:55 -0700 Subject: [PATCH 13/20] fix email notifications missing personalisation (notify-api-853) --- app/organization/rest.py | 9 +++++++++ app/user/rest.py | 27 +++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/app/organization/rest.py b/app/organization/rest.py index 7ca9eec8a..8da757cbc 100644 --- a/app/organization/rest.py +++ b/app/organization/rest.py @@ -1,6 +1,9 @@ +import json + from flask import Blueprint, abort, current_app, jsonify, request from sqlalchemy.exc import IntegrityError +from app import redis_store from app.config import QueueNames from app.dao.annual_billing_dao import set_default_free_allowance_for_service from app.dao.dao_utils import transaction @@ -210,6 +213,12 @@ def send_notifications_on_mou_signed(organization_id): reply_to_text=notify_service.get_default_reply_to_email_address(), ) saved_notification.personalisation = personalisation + + redis_store.set( + f"email-personalisation-{saved_notification.id}", + json.dumps(personalisation), + ex=60 * 60, + ) send_notification_to_queue(saved_notification, queue=QueueNames.NOTIFY) personalisation = { diff --git a/app/user/rest.py b/app/user/rest.py index 09ffa17c6..2cbbb9b25 100644 --- a/app/user/rest.py +++ b/app/user/rest.py @@ -140,6 +140,11 @@ def update_user_attribute(user_id): ) saved_notification.personalisation = personalisation + redis_store.set( + f"email-personalisation-{saved_notification.id}", + json.dumps(personalisation), + ex=60 * 60, + ) send_notification_to_queue(saved_notification, queue=QueueNames.NOTIFY) return jsonify(data=user_to_update.serialize()), 200 @@ -361,6 +366,12 @@ def create_2fa_code( # Assume that we never want to observe the Notify service's research mode # setting for this notification - we still need to be able to log into the # admin even if we're doing user research using this service: + + redis_store.set( + f"email-personalisation-{saved_notification.id}", + json.dumps(personalisation), + ex=60 * 60, + ) send_notification_to_queue(saved_notification, queue=QueueNames.NOTIFY) @@ -394,6 +405,11 @@ def send_user_confirm_new_email(user_id): ) saved_notification.personalisation = personalisation + redis_store.set( + f"email-personalisation-{saved_notification.id}", + json.dumps(personalisation), + ex=60 * 60, + ) send_notification_to_queue(saved_notification, queue=QueueNames.NOTIFY) return jsonify({}), 204 @@ -487,6 +503,12 @@ def send_already_registered_email(user_id): current_app.logger.info("Sending notification to queue") + redis_store.set( + f"email-personalisation-{saved_notification.id}", + json.dumps(personalisation), + ex=60 * 60, + ) + send_notification_to_queue(saved_notification, queue=QueueNames.NOTIFY) current_app.logger.info("Sent notification to queue") @@ -614,6 +636,11 @@ def send_user_reset_password(): ) saved_notification.personalisation = personalisation + redis_store.set( + f"email-personalisation-{saved_notification.id}", + json.dumps(personalisation), + ex=60 * 60, + ) send_notification_to_queue(saved_notification, queue=QueueNames.NOTIFY) return jsonify({}), 204 From 01df74160997839b2fe77b022c2dfe9c6f5ee56c Mon Sep 17 00:00:00 2001 From: Andrew Shumway Date: Wed, 13 Mar 2024 16:20:39 -0600 Subject: [PATCH 14/20] Add simulated numbers to allow list in trial mode --- app/service/utils.py | 4 ++++ tests/app/notifications/test_validators.py | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/app/service/utils.py b/app/service/utils.py index 2e22b5309..ac0613096 100644 --- a/app/service/utils.py +++ b/app/service/utils.py @@ -1,5 +1,6 @@ import itertools +from flask import current_app from notifications_utils.recipients import allowed_to_send_to from app.dao.services_dao import dao_fetch_service_by_id @@ -45,6 +46,9 @@ def service_allowed_to_send_to( member.recipient for member in service.guest_list if allow_guest_list_recipients ] + # As per discussion we have decided to allow official simulated + # numbers to go out in trial mode for development purposes. + guest_list_members.extend(current_app.config["SIMULATED_SMS_NUMBERS"]) if (key_type == KeyType.NORMAL and service.restricted) or ( key_type == KeyType.TEAM ): diff --git a/tests/app/notifications/test_validators.py b/tests/app/notifications/test_validators.py index f2d9cabb8..e16d5838e 100644 --- a/tests/app/notifications/test_validators.py +++ b/tests/app/notifications/test_validators.py @@ -34,6 +34,7 @@ from app.serialised_models import ( SerialisedService, SerialisedTemplate, ) +from app.service.utils import service_allowed_to_send_to from app.utils import get_template_instance from app.v2.errors import BadRequestError, RateLimitError, TotalRequestsError from tests.app.db import ( @@ -802,3 +803,21 @@ def test_check_service_over_total_message_limit(mocker, sample_service): sample_service, ) assert service_stats == 0 + + +def test_service_allowed_to_send_to_simulated_numbers(): + trial_mode_service = create_service(service_name="trial mode", restricted=True) + can_send = service_allowed_to_send_to( + "+14254147755", + trial_mode_service, + KeyType.NORMAL, + allow_guest_list_recipients=True, + ) + can_not_send = service_allowed_to_send_to( + "+15555555555", + trial_mode_service, + KeyType.NORMAL, + allow_guest_list_recipients=True, + ) + assert can_send is True + assert can_not_send is False From c7ff5e44c64c0d9fc15e11ac8ffda215e33c4f17 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 14 Mar 2024 11:27:26 -0700 Subject: [PATCH 15/20] duh --- poetry.lock | 20 ++++++++++---------- pyproject.toml | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/poetry.lock b/poetry.lock index 91c32b2b9..1dbab6175 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2657,11 +2657,11 @@ flask = "^2.3.2" flask-redis = "^0.4.0" geojson = "^3.0.1" govuk-bank-holidays = "^0.13" -idna = "^3.4" +idna = "^3.6" itsdangerous = "^2.1.2" jinja2 = "^3.1.3" jmespath = "^1.0.1" -markupsafe = "^2.1.2" +markupsafe = "^2.1.5" mistune = "==0.8.4" numpy = "^1.24.2" orderedset = "^2.0.3" @@ -2671,7 +2671,7 @@ python-dateutil = "^2.8.2" python-json-logger = "^2.0.7" pytz = "^2023.3" pyyaml = "^6.0" -redis = "^5.0.1" +redis = "^5.0.3" regex = "^2023.10.3" requests = "^2.31.0" s3transfer = "^0.7.0" @@ -2685,8 +2685,8 @@ werkzeug = "^3.0.1" [package.source] type = "git" url = "https://github.com/GSA/notifications-utils.git" -reference = "053fe30" -resolved_reference = "053fe304a1bbc1ae6559869b1fd0a2c3f83981ea" +reference = "HEAD" +resolved_reference = "e421fb1936d4bb92294a11fb78fc81b77d3577b2" [[package]] name = "numpy" @@ -3650,17 +3650,17 @@ full = ["numpy"] [[package]] name = "redis" -version = "5.0.2" +version = "5.0.3" description = "Python client for Redis database and key-value store" optional = false python-versions = ">=3.7" files = [ - {file = "redis-5.0.2-py3-none-any.whl", hash = "sha256:4caa8e1fcb6f3c0ef28dba99535101d80934b7d4cd541bbb47f4a3826ee472d1"}, - {file = "redis-5.0.2.tar.gz", hash = "sha256:3f82cc80d350e93042c8e6e7a5d0596e4dd68715babffba79492733e1f367037"}, + {file = "redis-5.0.3-py3-none-any.whl", hash = "sha256:5da9b8fe9e1254293756c16c008e8620b3d15fcc6dde6babde9541850e72a32d"}, + {file = "redis-5.0.3.tar.gz", hash = "sha256:4973bae7444c0fbed64a06b87446f79361cb7e4ec1538c022d696ed7a5015580"}, ] [package.dependencies] -async-timeout = ">=4.0.3" +async-timeout = {version = ">=4.0.3", markers = "python_full_version < \"3.11.3\""} [package.extras] hiredis = ["hiredis (>=1.0.0)"] @@ -4785,4 +4785,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.12" -content-hash = "dfa333e33a5f72123b0f8f37d76478d89569a5395c31b052eb867bb9ac40a1b7" +content-hash = "ed0a2c9f32e010a33a24e243a58550c4849d10ee8205f6c459ecdde57d3509ac" diff --git a/pyproject.toml b/pyproject.toml index 3a02d9b5f..4b8ab01ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,7 @@ marshmallow = "==3.20.2" marshmallow-sqlalchemy = "==0.30.0" newrelic = "*" notifications-python-client = "==9.0.0" -notifications-utils = {git = "https://github.com/GSA/notifications-utils.git", rev="053fe30"} +notifications-utils = {git = "https://github.com/GSA/notifications-utils.git"} oscrypto = "==1.3.0" packaging = "==23.2" poetry-dotenv-plugin = "==0.2.0" From da3c3f250847424d62e52fc28386d2528cd68cce Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 14 Mar 2024 19:15:37 +0000 Subject: [PATCH 16/20] Bump moto from 4.2.13 to 5.0.3 Bumps [moto](https://github.com/getmoto/moto) from 4.2.13 to 5.0.3. - [Release notes](https://github.com/getmoto/moto/releases) - [Changelog](https://github.com/getmoto/moto/blob/master/CHANGELOG.md) - [Commits](https://github.com/getmoto/moto/compare/4.2.13...5.0.3) --- updated-dependencies: - dependency-name: moto dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- poetry.lock | 48 +++++++++++++++++++++--------------------------- pyproject.toml | 2 +- 2 files changed, 22 insertions(+), 28 deletions(-) diff --git a/poetry.lock b/poetry.lock index 1dbab6175..7566ed1fb 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] name = "aiohttp" @@ -2339,50 +2339,44 @@ files = [ [[package]] name = "moto" -version = "4.2.13" +version = "5.0.3" description = "" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "moto-4.2.13-py2.py3-none-any.whl", hash = "sha256:93e0fd13b624bd79115494f833308c3641b2be0fc9f4f18aa9264aa01f6168e0"}, - {file = "moto-4.2.13.tar.gz", hash = "sha256:01aef6a489a725c8d725bd3dc6f70ff1bedaee3e2641752e4b471ff0ede4b4d7"}, + {file = "moto-5.0.3-py2.py3-none-any.whl", hash = "sha256:261d312d1d69c2afccb450a0566666d7b75d76ed6a7d00aac278a9633b073ff0"}, + {file = "moto-5.0.3.tar.gz", hash = "sha256:070ac2edf89ad7aee28534481ce68e2f344c8a6a8fefec5427eea0d599bfdbdb"}, ] [package.dependencies] boto3 = ">=1.9.201" -botocore = ">=1.12.201" +botocore = ">=1.14.0" cryptography = ">=3.3.1" Jinja2 = ">=2.10.1" python-dateutil = ">=2.1,<3.0.0" requests = ">=2.5" -responses = ">=0.13.0" +responses = ">=0.15.0" werkzeug = ">=0.5,<2.2.0 || >2.2.0,<2.2.1 || >2.2.1" xmltodict = "*" [package.extras] -all = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "ecdsa (!=0.15)", "graphql-core", "jsondiff (>=1.1.2)", "multipart", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.0)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] -apigateway = ["PyYAML (>=5.1)", "ecdsa (!=0.15)", "openapi-spec-validator (>=0.5.0)", "python-jose[cryptography] (>=3.1.0,<4.0.0)"] -apigatewayv2 = ["PyYAML (>=5.1)"] +all = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "graphql-core", "joserfc (>=0.9.0)", "jsondiff (>=1.1.2)", "multipart", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.1)", "pyparsing (>=3.0.7)", "setuptools"] +apigateway = ["PyYAML (>=5.1)", "joserfc (>=0.9.0)", "openapi-spec-validator (>=0.5.0)"] +apigatewayv2 = ["PyYAML (>=5.1)", "openapi-spec-validator (>=0.5.0)"] appsync = ["graphql-core"] awslambda = ["docker (>=3.0.0)"] batch = ["docker (>=3.0.0)"] -cloudformation = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "ecdsa (!=0.15)", "graphql-core", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.0)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] -cognitoidp = ["ecdsa (!=0.15)", "python-jose[cryptography] (>=3.1.0,<4.0.0)"] -ds = ["sshpubkeys (>=3.1.0)"] -dynamodb = ["docker (>=3.0.0)", "py-partiql-parser (==0.5.0)"] -dynamodbstreams = ["docker (>=3.0.0)", "py-partiql-parser (==0.5.0)"] -ebs = ["sshpubkeys (>=3.1.0)"] -ec2 = ["sshpubkeys (>=3.1.0)"] -efs = ["sshpubkeys (>=3.1.0)"] -eks = ["sshpubkeys (>=3.1.0)"] +cloudformation = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "graphql-core", "joserfc (>=0.9.0)", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.1)", "pyparsing (>=3.0.7)", "setuptools"] +cognitoidp = ["joserfc (>=0.9.0)"] +dynamodb = ["docker (>=3.0.0)", "py-partiql-parser (==0.5.1)"] +dynamodbstreams = ["docker (>=3.0.0)", "py-partiql-parser (==0.5.1)"] glue = ["pyparsing (>=3.0.7)"] iotdata = ["jsondiff (>=1.1.2)"] -proxy = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=2.5.1)", "ecdsa (!=0.15)", "graphql-core", "jsondiff (>=1.1.2)", "multipart", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.0)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] -resourcegroupstaggingapi = ["PyYAML (>=5.1)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "ecdsa (!=0.15)", "graphql-core", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.0)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "sshpubkeys (>=3.1.0)"] -route53resolver = ["sshpubkeys (>=3.1.0)"] -s3 = ["PyYAML (>=5.1)", "py-partiql-parser (==0.5.0)"] -s3crc32c = ["PyYAML (>=5.1)", "crc32c", "py-partiql-parser (==0.5.0)"] -server = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "ecdsa (!=0.15)", "flask (!=2.2.0,!=2.2.1)", "flask-cors", "graphql-core", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.0)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] +proxy = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=2.5.1)", "graphql-core", "joserfc (>=0.9.0)", "jsondiff (>=1.1.2)", "multipart", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.1)", "pyparsing (>=3.0.7)", "setuptools"] +resourcegroupstaggingapi = ["PyYAML (>=5.1)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "graphql-core", "joserfc (>=0.9.0)", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.1)", "pyparsing (>=3.0.7)"] +s3 = ["PyYAML (>=5.1)", "py-partiql-parser (==0.5.1)"] +s3crc32c = ["PyYAML (>=5.1)", "crc32c", "py-partiql-parser (==0.5.1)"] +server = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "flask (!=2.2.0,!=2.2.1)", "flask-cors", "graphql-core", "joserfc (>=0.9.0)", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.1)", "pyparsing (>=3.0.7)", "setuptools"] ssm = ["PyYAML (>=5.1)"] xray = ["aws-xray-sdk (>=0.93,!=0.96)", "setuptools"] @@ -2448,7 +2442,6 @@ files = [ {file = "msgpack-1.0.8-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5fbb160554e319f7b22ecf530a80a3ff496d38e8e07ae763b9e82fadfe96f273"}, {file = "msgpack-1.0.8-cp39-cp39-win32.whl", hash = "sha256:f9af38a89b6a5c04b7d18c492c8ccf2aee7048aff1ce8437c4683bb5a1df893d"}, {file = "msgpack-1.0.8-cp39-cp39-win_amd64.whl", hash = "sha256:ed59dd52075f8fc91da6053b12e8c89e37aa043f8986efd89e61fae69dc1b011"}, - {file = "msgpack-1.0.8-py3-none-any.whl", hash = "sha256:24f727df1e20b9876fa6e95f840a2a2651e34c0ad147676356f4bf5fbb0206ca"}, {file = "msgpack-1.0.8.tar.gz", hash = "sha256:95c02b0e27e706e48d0e5426d1710ca78e0f0628d6e89d5b5a5b91a5f12274f3"}, ] @@ -3494,6 +3487,7 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, @@ -4785,4 +4779,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.12" -content-hash = "ed0a2c9f32e010a33a24e243a58550c4849d10ee8205f6c459ecdde57d3509ac" +content-hash = "4f1042e13a7102d29b8580dc80a400907dd596d243318217e5dd28686c3a1f19" diff --git a/pyproject.toml b/pyproject.toml index 4b8ab01ef..434672868 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,7 +65,7 @@ freezegun = "^1.4.0" honcho = "*" isort = "^5.13.2" jinja2-cli = {version = "==0.8.2", extras = ["yaml"]} -moto = "==4.2.13" +moto = "==5.0.3" pip-audit = "*" pre-commit = "^3.6.0" pytest = "^7.4.4" From 435f23b5f0fcfce47d10dba3c105514fabb4538f Mon Sep 17 00:00:00 2001 From: Carlo Costino Date: Thu, 14 Mar 2024 15:15:51 -0400 Subject: [PATCH 17/20] This changeset updates notifications_utils to 0.3.0 to account for other dependency updates. It also accounts for the removal of Celery in utils. Signed-off-by: Carlo Costino --- poetry.lock | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/poetry.lock b/poetry.lock index 1dbab6175..117555c6c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2036,7 +2036,6 @@ description = "Powerful and Pythonic XML processing library combining libxml2/li optional = false python-versions = ">=3.6" files = [ - {file = "lxml-5.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:704f5572ff473a5f897745abebc6df40f22d4133c1e0a1f124e4f2bd3330ff7e"}, {file = "lxml-5.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9d3c0f8567ffe7502d969c2c1b809892dc793b5d0665f602aad19895f8d508da"}, {file = "lxml-5.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5fcfbebdb0c5d8d18b84118842f31965d59ee3e66996ac842e21f957eb76138c"}, {file = "lxml-5.1.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2f37c6d7106a9d6f0708d4e164b707037b7380fcd0b04c5bd9cae1fb46a856fb"}, @@ -2046,7 +2045,6 @@ files = [ {file = "lxml-5.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:82bddf0e72cb2af3cbba7cec1d2fd11fda0de6be8f4492223d4a268713ef2147"}, {file = "lxml-5.1.0-cp310-cp310-win32.whl", hash = "sha256:b66aa6357b265670bb574f050ffceefb98549c721cf28351b748be1ef9577d93"}, {file = "lxml-5.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:4946e7f59b7b6a9e27bef34422f645e9a368cb2be11bf1ef3cafc39a1f6ba68d"}, - {file = "lxml-5.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:14deca1460b4b0f6b01f1ddc9557704e8b365f55c63070463f6c18619ebf964f"}, {file = "lxml-5.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ed8c3d2cd329bf779b7ed38db176738f3f8be637bb395ce9629fc76f78afe3d4"}, {file = "lxml-5.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:436a943c2900bb98123b06437cdd30580a61340fbdb7b28aaf345a459c19046a"}, {file = "lxml-5.1.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:acb6b2f96f60f70e7f34efe0c3ea34ca63f19ca63ce90019c6cbca6b676e81fa"}, @@ -2056,7 +2054,6 @@ files = [ {file = "lxml-5.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f4c9bda132ad108b387c33fabfea47866af87f4ea6ffb79418004f0521e63204"}, {file = "lxml-5.1.0-cp311-cp311-win32.whl", hash = "sha256:bc64d1b1dab08f679fb89c368f4c05693f58a9faf744c4d390d7ed1d8223869b"}, {file = "lxml-5.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:a5ab722ae5a873d8dcee1f5f45ddd93c34210aed44ff2dc643b5025981908cda"}, - {file = "lxml-5.1.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:9aa543980ab1fbf1720969af1d99095a548ea42e00361e727c58a40832439114"}, {file = "lxml-5.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:6f11b77ec0979f7e4dc5ae081325a2946f1fe424148d3945f943ceaede98adb8"}, {file = "lxml-5.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a36c506e5f8aeb40680491d39ed94670487ce6614b9d27cabe45d94cd5d63e1e"}, {file = "lxml-5.1.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f643ffd2669ffd4b5a3e9b41c909b72b2a1d5e4915da90a77e119b8d48ce867a"}, @@ -2082,8 +2079,8 @@ files = [ {file = "lxml-5.1.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8f52fe6859b9db71ee609b0c0a70fea5f1e71c3462ecf144ca800d3f434f0764"}, {file = "lxml-5.1.0-cp37-cp37m-win32.whl", hash = "sha256:d42e3a3fc18acc88b838efded0e6ec3edf3e328a58c68fbd36a7263a874906c8"}, {file = "lxml-5.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:eac68f96539b32fce2c9b47eb7c25bb2582bdaf1bbb360d25f564ee9e04c542b"}, - {file = "lxml-5.1.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:ae15347a88cf8af0949a9872b57a320d2605ae069bcdf047677318bc0bba45b1"}, {file = "lxml-5.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c26aab6ea9c54d3bed716b8851c8bfc40cb249b8e9880e250d1eddde9f709bf5"}, + {file = "lxml-5.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:cfbac9f6149174f76df7e08c2e28b19d74aed90cad60383ad8671d3af7d0502f"}, {file = "lxml-5.1.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:342e95bddec3a698ac24378d61996b3ee5ba9acfeb253986002ac53c9a5f6f84"}, {file = "lxml-5.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:725e171e0b99a66ec8605ac77fa12239dbe061482ac854d25720e2294652eeaa"}, {file = "lxml-5.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d184e0d5c918cff04cdde9dbdf9600e960161d773666958c9d7b565ccc60c45"}, @@ -2091,7 +2088,6 @@ files = [ {file = "lxml-5.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6d48fc57e7c1e3df57be5ae8614bab6d4e7b60f65c5457915c26892c41afc59e"}, {file = "lxml-5.1.0-cp38-cp38-win32.whl", hash = "sha256:7ec465e6549ed97e9f1e5ed51c657c9ede767bc1c11552f7f4d022c4df4a977a"}, {file = "lxml-5.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:b21b4031b53d25b0858d4e124f2f9131ffc1530431c6d1321805c90da78388d1"}, - {file = "lxml-5.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:52427a7eadc98f9e62cb1368a5079ae826f94f05755d2d567d93ee1bc3ceb354"}, {file = "lxml-5.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6a2a2c724d97c1eb8cf966b16ca2915566a4904b9aad2ed9a09c748ffe14f969"}, {file = "lxml-5.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:843b9c835580d52828d8f69ea4302537337a21e6b4f1ec711a52241ba4a824f3"}, {file = "lxml-5.1.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9b99f564659cfa704a2dd82d0684207b1aadf7d02d33e54845f9fc78e06b7581"}, @@ -2448,7 +2444,6 @@ files = [ {file = "msgpack-1.0.8-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5fbb160554e319f7b22ecf530a80a3ff496d38e8e07ae763b9e82fadfe96f273"}, {file = "msgpack-1.0.8-cp39-cp39-win32.whl", hash = "sha256:f9af38a89b6a5c04b7d18c492c8ccf2aee7048aff1ce8437c4683bb5a1df893d"}, {file = "msgpack-1.0.8-cp39-cp39-win_amd64.whl", hash = "sha256:ed59dd52075f8fc91da6053b12e8c89e37aa043f8986efd89e61fae69dc1b011"}, - {file = "msgpack-1.0.8-py3-none-any.whl", hash = "sha256:24f727df1e20b9876fa6e95f840a2a2651e34c0ad147676356f4bf5fbb0206ca"}, {file = "msgpack-1.0.8.tar.gz", hash = "sha256:95c02b0e27e706e48d0e5426d1710ca78e0f0628d6e89d5b5a5b91a5f12274f3"}, ] @@ -2634,7 +2629,7 @@ requests = ">=2.0.0" [[package]] name = "notifications-utils" -version = "0.2.9" +version = "0.3.0" description = "" optional = false python-versions = ">=3.9,<3.12" @@ -2686,7 +2681,7 @@ werkzeug = "^3.0.1" type = "git" url = "https://github.com/GSA/notifications-utils.git" reference = "HEAD" -resolved_reference = "e421fb1936d4bb92294a11fb78fc81b77d3577b2" +resolved_reference = "5a623566c7f71db48699cee93415f45317541524" [[package]] name = "numpy" From fee88b86d226c38e9bf1095fdf66448d51634861 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 14 Mar 2024 21:18:07 +0000 Subject: [PATCH 18/20] Bump cachetools from 5.3.2 to 5.3.3 Bumps [cachetools](https://github.com/tkem/cachetools) from 5.3.2 to 5.3.3. - [Changelog](https://github.com/tkem/cachetools/blob/master/CHANGELOG.rst) - [Commits](https://github.com/tkem/cachetools/compare/v5.3.2...v5.3.3) --- updated-dependencies: - dependency-name: cachetools dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- poetry.lock | 14 +++++++++----- pyproject.toml | 2 +- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/poetry.lock b/poetry.lock index f0f3cba1f..02d818868 100644 --- a/poetry.lock +++ b/poetry.lock @@ -491,13 +491,13 @@ redis = ["redis (>=2.10.5)"] [[package]] name = "cachetools" -version = "5.3.2" +version = "5.3.3" description = "Extensible memoizing collections and decorators" optional = false python-versions = ">=3.7" files = [ - {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, - {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, + {file = "cachetools-5.3.3-py3-none-any.whl", hash = "sha256:0abad1021d3f8325b2fc1d2e9c8b9c9d57b04c3932657a72465447332c24d945"}, + {file = "cachetools-5.3.3.tar.gz", hash = "sha256:ba29e2dfa0b8b556606f097407ed1aa62080ee108ab0dc5ec9d6a723a007d105"}, ] [[package]] @@ -2036,6 +2036,7 @@ description = "Powerful and Pythonic XML processing library combining libxml2/li optional = false python-versions = ">=3.6" files = [ + {file = "lxml-5.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:704f5572ff473a5f897745abebc6df40f22d4133c1e0a1f124e4f2bd3330ff7e"}, {file = "lxml-5.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9d3c0f8567ffe7502d969c2c1b809892dc793b5d0665f602aad19895f8d508da"}, {file = "lxml-5.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5fcfbebdb0c5d8d18b84118842f31965d59ee3e66996ac842e21f957eb76138c"}, {file = "lxml-5.1.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2f37c6d7106a9d6f0708d4e164b707037b7380fcd0b04c5bd9cae1fb46a856fb"}, @@ -2045,6 +2046,7 @@ files = [ {file = "lxml-5.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:82bddf0e72cb2af3cbba7cec1d2fd11fda0de6be8f4492223d4a268713ef2147"}, {file = "lxml-5.1.0-cp310-cp310-win32.whl", hash = "sha256:b66aa6357b265670bb574f050ffceefb98549c721cf28351b748be1ef9577d93"}, {file = "lxml-5.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:4946e7f59b7b6a9e27bef34422f645e9a368cb2be11bf1ef3cafc39a1f6ba68d"}, + {file = "lxml-5.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:14deca1460b4b0f6b01f1ddc9557704e8b365f55c63070463f6c18619ebf964f"}, {file = "lxml-5.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ed8c3d2cd329bf779b7ed38db176738f3f8be637bb395ce9629fc76f78afe3d4"}, {file = "lxml-5.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:436a943c2900bb98123b06437cdd30580a61340fbdb7b28aaf345a459c19046a"}, {file = "lxml-5.1.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:acb6b2f96f60f70e7f34efe0c3ea34ca63f19ca63ce90019c6cbca6b676e81fa"}, @@ -2054,6 +2056,7 @@ files = [ {file = "lxml-5.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f4c9bda132ad108b387c33fabfea47866af87f4ea6ffb79418004f0521e63204"}, {file = "lxml-5.1.0-cp311-cp311-win32.whl", hash = "sha256:bc64d1b1dab08f679fb89c368f4c05693f58a9faf744c4d390d7ed1d8223869b"}, {file = "lxml-5.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:a5ab722ae5a873d8dcee1f5f45ddd93c34210aed44ff2dc643b5025981908cda"}, + {file = "lxml-5.1.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:9aa543980ab1fbf1720969af1d99095a548ea42e00361e727c58a40832439114"}, {file = "lxml-5.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:6f11b77ec0979f7e4dc5ae081325a2946f1fe424148d3945f943ceaede98adb8"}, {file = "lxml-5.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a36c506e5f8aeb40680491d39ed94670487ce6614b9d27cabe45d94cd5d63e1e"}, {file = "lxml-5.1.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f643ffd2669ffd4b5a3e9b41c909b72b2a1d5e4915da90a77e119b8d48ce867a"}, @@ -2079,8 +2082,8 @@ files = [ {file = "lxml-5.1.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8f52fe6859b9db71ee609b0c0a70fea5f1e71c3462ecf144ca800d3f434f0764"}, {file = "lxml-5.1.0-cp37-cp37m-win32.whl", hash = "sha256:d42e3a3fc18acc88b838efded0e6ec3edf3e328a58c68fbd36a7263a874906c8"}, {file = "lxml-5.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:eac68f96539b32fce2c9b47eb7c25bb2582bdaf1bbb360d25f564ee9e04c542b"}, + {file = "lxml-5.1.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:ae15347a88cf8af0949a9872b57a320d2605ae069bcdf047677318bc0bba45b1"}, {file = "lxml-5.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c26aab6ea9c54d3bed716b8851c8bfc40cb249b8e9880e250d1eddde9f709bf5"}, - {file = "lxml-5.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:cfbac9f6149174f76df7e08c2e28b19d74aed90cad60383ad8671d3af7d0502f"}, {file = "lxml-5.1.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:342e95bddec3a698ac24378d61996b3ee5ba9acfeb253986002ac53c9a5f6f84"}, {file = "lxml-5.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:725e171e0b99a66ec8605ac77fa12239dbe061482ac854d25720e2294652eeaa"}, {file = "lxml-5.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d184e0d5c918cff04cdde9dbdf9600e960161d773666958c9d7b565ccc60c45"}, @@ -2088,6 +2091,7 @@ files = [ {file = "lxml-5.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6d48fc57e7c1e3df57be5ae8614bab6d4e7b60f65c5457915c26892c41afc59e"}, {file = "lxml-5.1.0-cp38-cp38-win32.whl", hash = "sha256:7ec465e6549ed97e9f1e5ed51c657c9ede767bc1c11552f7f4d022c4df4a977a"}, {file = "lxml-5.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:b21b4031b53d25b0858d4e124f2f9131ffc1530431c6d1321805c90da78388d1"}, + {file = "lxml-5.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:52427a7eadc98f9e62cb1368a5079ae826f94f05755d2d567d93ee1bc3ceb354"}, {file = "lxml-5.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6a2a2c724d97c1eb8cf966b16ca2915566a4904b9aad2ed9a09c748ffe14f969"}, {file = "lxml-5.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:843b9c835580d52828d8f69ea4302537337a21e6b4f1ec711a52241ba4a824f3"}, {file = "lxml-5.1.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9b99f564659cfa704a2dd82d0684207b1aadf7d02d33e54845f9fc78e06b7581"}, @@ -4775,4 +4779,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.12" -content-hash = "4f1042e13a7102d29b8580dc80a400907dd596d243318217e5dd28686c3a1f19" +content-hash = "2d2d43a92e1f61a653f5308d68bee42aea35722f546880f430c2233fd5cf3724" diff --git a/pyproject.toml b/pyproject.toml index 434672868..d64610671 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ amqp = "==5.2.0" beautifulsoup4 = "==4.12.3" boto3 = "^1.29.6" botocore = "^1.32.6" -cachetools = "==5.3.2" +cachetools = "==5.3.3" celery = {version = "==5.3.6", extras = ["redis"]} certifi = ">=2022.12.7" cffi = "==1.16.0" From bf0f002e4fb7193cced16ddeb5b8562ce286e456 Mon Sep 17 00:00:00 2001 From: Carlo Costino Date: Fri, 15 Mar 2024 14:26:18 -0400 Subject: [PATCH 19/20] Remove debug lines in CSV processing This changeset removes a couple of debug lines that we had in place from when we switched to processing CSVs for looking at phone numbers. This has now been proven out, and the additional log entries make it difficult to troubleshoot other issues that may arise. Signed-off-by: Carlo Costino --- app/aws/s3.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/aws/s3.py b/app/aws/s3.py index d4c033a63..f80bf6f71 100644 --- a/app/aws/s3.py +++ b/app/aws/s3.py @@ -114,8 +114,7 @@ def extract_phones(job): job_row = 0 for row in job: row = row.split(",") - current_app.logger.info(f"PHONE INDEX IS NOW {phone_index}") - current_app.logger.info(f"LENGTH OF ROW IS {len(row)}") + if phone_index >= len(row): phones[job_row] = "Unavailable" current_app.logger.error( From 117fa6c1af07582ea22bf474a2e1e7279a3adfa4 Mon Sep 17 00:00:00 2001 From: Carlo Costino Date: Fri, 15 Mar 2024 16:16:58 -0400 Subject: [PATCH 20/20] Adjust production worker memory to 1G This changeset adjusts our worker process memory to 1G from 512MB in production. We recently saw memory-related crashes for the Celery worker processes, meaning they did not have enough available to them. This was compounded by an ongoing platform issue with cloud.gov due to Cloud Foundry VMs operating with a Linux kernel that has a memory allocation issue. Fixes for the Linux kernel and Cloud Foundry VMs are in flight and cloud.gov is tracking this closely, but we can help ourselves in the mean time and given the increased usage with pilot partners, it makes sense to adjust this anyway. Signed-off-by: Carlo Costino --- deploy-config/production.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy-config/production.yml b/deploy-config/production.yml index 2152eae42..18ed2a0c3 100644 --- a/deploy-config/production.yml +++ b/deploy-config/production.yml @@ -2,7 +2,7 @@ env: production web_instances: 2 web_memory: 1G worker_instances: 1 -worker_memory: 512M +worker_memory: 1G scheduler_memory: 256M public_api_route: notify-api.app.cloud.gov admin_base_url: https://beta.notify.gov