diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 280d2566e..dd0fe3275 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -61,7 +61,8 @@ jobs: NOTIFY_E2E_TEST_HTTP_AUTH_USER: ${{ secrets.NOTIFY_E2E_TEST_HTTP_AUTH_USER }} NOTIFY_E2E_TEST_PASSWORD: ${{ secrets.NOTIFY_E2E_TEST_PASSWORD }} - name: Check coverage threshold - run: poetry run coverage report --fail-under=50 + # TODO get this back up to 95 + run: poetry run coverage report --fail-under=87 validate-new-relic-config: runs-on: ubuntu-latest diff --git a/Makefile b/Makefile index 4273f7b7e..e86cfc1c3 100644 --- a/Makefile +++ b/Makefile @@ -79,7 +79,7 @@ test: ## Run tests and create coverage report poetry run black . poetry run flake8 . poetry run isort --check-only ./app ./tests - poetry run coverage run -m pytest -vv --maxfail=10 + poetry run coverage run --omit=*/notifications_utils/* -m pytest --maxfail=10 poetry run coverage report -m --fail-under=95 poetry run coverage html -d .coverage_cache diff --git a/app/organization/rest.py b/app/organization/rest.py index 7ca9eec8a..8da757cbc 100644 --- a/app/organization/rest.py +++ b/app/organization/rest.py @@ -1,6 +1,9 @@ +import json + from flask import Blueprint, abort, current_app, jsonify, request from sqlalchemy.exc import IntegrityError +from app import redis_store from app.config import QueueNames from app.dao.annual_billing_dao import set_default_free_allowance_for_service from app.dao.dao_utils import transaction @@ -210,6 +213,12 @@ def send_notifications_on_mou_signed(organization_id): reply_to_text=notify_service.get_default_reply_to_email_address(), ) saved_notification.personalisation = personalisation + + redis_store.set( + f"email-personalisation-{saved_notification.id}", + json.dumps(personalisation), + ex=60 * 60, + ) send_notification_to_queue(saved_notification, queue=QueueNames.NOTIFY) personalisation = { diff --git a/app/user/rest.py b/app/user/rest.py index 09ffa17c6..2cbbb9b25 100644 --- a/app/user/rest.py +++ b/app/user/rest.py @@ -140,6 +140,11 @@ def update_user_attribute(user_id): ) saved_notification.personalisation = personalisation + redis_store.set( + f"email-personalisation-{saved_notification.id}", + json.dumps(personalisation), + ex=60 * 60, + ) send_notification_to_queue(saved_notification, queue=QueueNames.NOTIFY) return jsonify(data=user_to_update.serialize()), 200 @@ -361,6 +366,12 @@ def create_2fa_code( # Assume that we never want to observe the Notify service's research mode # setting for this notification - we still need to be able to log into the # admin even if we're doing user research using this service: + + redis_store.set( + f"email-personalisation-{saved_notification.id}", + json.dumps(personalisation), + ex=60 * 60, + ) send_notification_to_queue(saved_notification, queue=QueueNames.NOTIFY) @@ -394,6 +405,11 @@ def send_user_confirm_new_email(user_id): ) saved_notification.personalisation = personalisation + redis_store.set( + f"email-personalisation-{saved_notification.id}", + json.dumps(personalisation), + ex=60 * 60, + ) send_notification_to_queue(saved_notification, queue=QueueNames.NOTIFY) return jsonify({}), 204 @@ -487,6 +503,12 @@ def send_already_registered_email(user_id): current_app.logger.info("Sending notification to queue") + redis_store.set( + f"email-personalisation-{saved_notification.id}", + json.dumps(personalisation), + ex=60 * 60, + ) + send_notification_to_queue(saved_notification, queue=QueueNames.NOTIFY) current_app.logger.info("Sent notification to queue") @@ -614,6 +636,11 @@ def send_user_reset_password(): ) saved_notification.personalisation = personalisation + redis_store.set( + f"email-personalisation-{saved_notification.id}", + json.dumps(personalisation), + ex=60 * 60, + ) send_notification_to_queue(saved_notification, queue=QueueNames.NOTIFY) return jsonify({}), 204 diff --git a/migrations/versions/0410_enums_for_everything.py b/migrations/versions/0410_enums_for_everything.py index e34a3621a..b6c9042c6 100644 --- a/migrations/versions/0410_enums_for_everything.py +++ b/migrations/versions/0410_enums_for_everything.py @@ -468,6 +468,19 @@ def upgrade(): existing_nullable=False, postgresql_using=enum_using("notification_type", NotificationType), ) + # Clobbering bad data here. These are values we don't use any more, and anything with them is unnecessary. + op.execute(""" + delete from + service_permissions + where + permission in ( + 'letter', + 'letters_as_pdf', + 'upload_letters', + 'international_letters', + 'broadcast' + ); + """) op.alter_column( "service_permissions", "permission", diff --git a/terraform/demo/main.tf b/terraform/demo/main.tf index 615f92670..e594264c2 100644 --- a/terraform/demo/main.tf +++ b/terraform/demo/main.tf @@ -1,38 +1,45 @@ locals { - cf_org_name = "gsa-tts-benefits-studio" - cf_space_name = "notify-demo" - env = "demo" - app_name = "notify-api" - recursive_delete = false + cf_org_name = "gsa-tts-benefits-studio" + cf_space_name = "notify-demo" + env = "demo" + app_name = "notify-api" + delete_recursive_allowed = false +} + +data "cloudfoundry_org" "org" { + name = local.cf_org_name +} + +resource "cloudfoundry_space" "notify-demo" { + delete_recursive_allowed = local.delete_recursive_allowed + name = local.cf_space_name + org = data.cloudfoundry_org.org.id } module "database" { source = "github.com/18f/terraform-cloudgov//database?ref=v0.7.1" - cf_org_name = local.cf_org_name - cf_space_name = local.cf_space_name - name = "${local.app_name}-rds-${local.env}" - recursive_delete = local.recursive_delete - rds_plan_name = "micro-psql" + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + name = "${local.app_name}-rds-${local.env}" + rds_plan_name = "micro-psql" } module "redis" { source = "github.com/18f/terraform-cloudgov//redis?ref=v0.7.1" - cf_org_name = local.cf_org_name - cf_space_name = local.cf_space_name - name = "${local.app_name}-redis-${local.env}" - recursive_delete = local.recursive_delete - redis_plan_name = "redis-dev" + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + name = "${local.app_name}-redis-${local.env}" + redis_plan_name = "redis-dev" } module "csv_upload_bucket" { source = "github.com/18f/terraform-cloudgov//s3?ref=v0.7.1" - cf_org_name = local.cf_org_name - cf_space_name = local.cf_space_name - recursive_delete = local.recursive_delete - name = "${local.app_name}-csv-upload-bucket-${local.env}" + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + name = "${local.app_name}-csv-upload-bucket-${local.env}" } module "egress-space" { @@ -40,6 +47,7 @@ module "egress-space" { cf_org_name = local.cf_org_name cf_restricted_space_name = local.cf_space_name + delete_recursive_allowed = local.delete_recursive_allowed deployers = [ var.cf_user, "steven.reilly@gsa.gov" @@ -52,7 +60,6 @@ module "ses_email" { cf_org_name = local.cf_org_name cf_space_name = local.cf_space_name name = "${local.app_name}-ses-${local.env}" - recursive_delete = local.recursive_delete aws_region = "us-west-2" email_domain = "notify.sandbox.10x.gsa.gov" email_receipt_error = "notify-support@gsa.gov" @@ -64,7 +71,6 @@ module "sns_sms" { cf_org_name = local.cf_org_name cf_space_name = local.cf_space_name name = "${local.app_name}-sns-${local.env}" - recursive_delete = local.recursive_delete aws_region = "us-east-1" monthly_spend_limit = 25 } diff --git a/terraform/demo/providers.tf b/terraform/demo/providers.tf index f13333d3e..34ba30a62 100644 --- a/terraform/demo/providers.tf +++ b/terraform/demo/providers.tf @@ -3,7 +3,7 @@ terraform { required_providers { cloudfoundry = { source = "cloudfoundry-community/cloudfoundry" - version = "0.53.0" + version = "0.53.1" } } diff --git a/terraform/production/main.tf b/terraform/production/main.tf index 5a2c520b1..ff1daad88 100644 --- a/terraform/production/main.tf +++ b/terraform/production/main.tf @@ -1,45 +1,56 @@ locals { - cf_org_name = "gsa-tts-benefits-studio" - cf_space_name = "notify-production" - env = "production" - app_name = "notify-api" - recursive_delete = false + cf_org_name = "gsa-tts-benefits-studio" + cf_space_name = "notify-production" + env = "production" + app_name = "notify-api" + delete_recursive_allowed = false + allow_ssh = false +} + +data "cloudfoundry_org" "org" { + name = local.cf_org_name +} + +resource "cloudfoundry_space" "notify-production" { + allow_ssh = local.allow_ssh + delete_recursive_allowed = local.delete_recursive_allowed + name = local.cf_space_name + org = data.cloudfoundry_org.org.id } module "database" { source = "github.com/18f/terraform-cloudgov//database?ref=v0.7.1" - cf_org_name = local.cf_org_name - cf_space_name = local.cf_space_name - name = "${local.app_name}-rds-${local.env}" - recursive_delete = local.recursive_delete - rds_plan_name = "small-psql-redundant" + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + name = "${local.app_name}-rds-${local.env}" + rds_plan_name = "small-psql-redundant" } module "redis" { source = "github.com/18f/terraform-cloudgov//redis?ref=v0.7.1" - cf_org_name = local.cf_org_name - cf_space_name = local.cf_space_name - name = "${local.app_name}-redis-${local.env}" - recursive_delete = local.recursive_delete - redis_plan_name = "redis-3node-large" + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + name = "${local.app_name}-redis-${local.env}" + redis_plan_name = "redis-3node-large" } module "csv_upload_bucket" { source = "github.com/18f/terraform-cloudgov//s3?ref=v0.7.1" - cf_org_name = local.cf_org_name - cf_space_name = local.cf_space_name - recursive_delete = local.recursive_delete - name = "${local.app_name}-csv-upload-bucket-${local.env}" + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + name = "${local.app_name}-csv-upload-bucket-${local.env}" } module "egress-space" { source = "../shared/egress_space" + allow_ssh = local.allow_ssh cf_org_name = local.cf_org_name cf_restricted_space_name = local.cf_space_name + delete_recursive_allowed = local.delete_recursive_allowed deployers = [ var.cf_user ] @@ -51,7 +62,6 @@ module "ses_email" { cf_org_name = local.cf_org_name cf_space_name = local.cf_space_name name = "${local.app_name}-ses-${local.env}" - recursive_delete = local.recursive_delete aws_region = "us-gov-west-1" email_domain = "notify.gov" mail_from_subdomain = "mail" @@ -64,7 +74,6 @@ module "sns_sms" { cf_org_name = local.cf_org_name cf_space_name = local.cf_space_name name = "${local.app_name}-sns-${local.env}" - recursive_delete = local.recursive_delete aws_region = "us-gov-west-1" monthly_spend_limit = 1000 } diff --git a/terraform/production/providers.tf b/terraform/production/providers.tf index 499759f48..b5c45f63e 100644 --- a/terraform/production/providers.tf +++ b/terraform/production/providers.tf @@ -3,7 +3,7 @@ terraform { required_providers { cloudfoundry = { source = "cloudfoundry-community/cloudfoundry" - version = "0.53.0" + version = "0.53.1" } } diff --git a/terraform/shared/egress_space/main.tf b/terraform/shared/egress_space/main.tf index 5d4b53354..cc91e9c42 100644 --- a/terraform/shared/egress_space/main.tf +++ b/terraform/shared/egress_space/main.tf @@ -11,7 +11,8 @@ data "cloudfoundry_org" "org" { ### resource "cloudfoundry_space" "public_egress" { - delete_recursive_allowed = false + allow_ssh = var.allow_ssh + delete_recursive_allowed = var.delete_recursive_allowed name = "${var.cf_restricted_space_name}-egress" org = data.cloudfoundry_org.org.id } diff --git a/terraform/shared/egress_space/variables.tf b/terraform/shared/egress_space/variables.tf index 45bcc717d..5bdff893f 100644 --- a/terraform/shared/egress_space/variables.tf +++ b/terraform/shared/egress_space/variables.tf @@ -3,3 +3,15 @@ variable "cf_restricted_space_name" {} variable "deployers" { type = set(string) } + +variable "delete_recursive_allowed" { + type = bool + default = true + description = "Flag for allowing resources to be recursively deleted - not recommended in production environments" +} + +variable "allow_ssh" { + type = bool + default = true + description = "Flag for allowing SSH access in a space - not recommended in production environments" +}