Merge branch 'main' of https://github.com/GSA/notifications-api into add-simulated-to-local

.github/workflows/checks.yml

@@ -61,7 +61,8 @@ jobs:
         NOTIFY_E2E_TEST_HTTP_AUTH_USER: ${{ secrets.NOTIFY_E2E_TEST_HTTP_AUTH_USER }}
         NOTIFY_E2E_TEST_PASSWORD: ${{ secrets.NOTIFY_E2E_TEST_PASSWORD }}
     - name: Check coverage threshold
-      run: poetry run coverage report --fail-under=50
+      # TODO get this back up to 95
+      run: poetry run coverage report --fail-under=87
 
   validate-new-relic-config:
     runs-on: ubuntu-latest

Makefile

@@ -79,7 +79,7 @@ test: ## Run tests and create coverage report
 	poetry run black .
 	poetry run flake8 .
 	poetry run isort --check-only ./app ./tests
-	poetry run coverage run -m pytest -vv --maxfail=10
+	poetry run coverage run --omit=*/notifications_utils/* -m pytest --maxfail=10
 	poetry run coverage report -m --fail-under=95
 	poetry run coverage html -d .coverage_cache
 

app/organization/rest.py

@@ -1,6 +1,9 @@
+import json
+
 from flask import Blueprint, abort, current_app, jsonify, request
 from sqlalchemy.exc import IntegrityError
 
+from app import redis_store
 from app.config import QueueNames
 from app.dao.annual_billing_dao import set_default_free_allowance_for_service
 from app.dao.dao_utils import transaction
@@ -210,6 +213,12 @@ def send_notifications_on_mou_signed(organization_id):
             reply_to_text=notify_service.get_default_reply_to_email_address(),
         )
         saved_notification.personalisation = personalisation
+
+        redis_store.set(
+            f"email-personalisation-{saved_notification.id}",
+            json.dumps(personalisation),
+            ex=60 * 60,
+        )
         send_notification_to_queue(saved_notification, queue=QueueNames.NOTIFY)
 
     personalisation = {

app/user/rest.py

@@ -140,6 +140,11 @@ def update_user_attribute(user_id):
         )
         saved_notification.personalisation = personalisation
 
+        redis_store.set(
+            f"email-personalisation-{saved_notification.id}",
+            json.dumps(personalisation),
+            ex=60 * 60,
+        )
         send_notification_to_queue(saved_notification, queue=QueueNames.NOTIFY)
 
     return jsonify(data=user_to_update.serialize()), 200
@@ -361,6 +366,12 @@ def create_2fa_code(
     # Assume that we never want to observe the Notify service's research mode
     # setting for this notification - we still need to be able to log into the
     # admin even if we're doing user research using this service:
+
+    redis_store.set(
+        f"email-personalisation-{saved_notification.id}",
+        json.dumps(personalisation),
+        ex=60 * 60,
+    )
     send_notification_to_queue(saved_notification, queue=QueueNames.NOTIFY)
 
 
@@ -394,6 +405,11 @@ def send_user_confirm_new_email(user_id):
     )
     saved_notification.personalisation = personalisation
 
+    redis_store.set(
+        f"email-personalisation-{saved_notification.id}",
+        json.dumps(personalisation),
+        ex=60 * 60,
+    )
     send_notification_to_queue(saved_notification, queue=QueueNames.NOTIFY)
     return jsonify({}), 204
 
@@ -487,6 +503,12 @@ def send_already_registered_email(user_id):
 
     current_app.logger.info("Sending notification to queue")
 
+    redis_store.set(
+        f"email-personalisation-{saved_notification.id}",
+        json.dumps(personalisation),
+        ex=60 * 60,
+    )
+
     send_notification_to_queue(saved_notification, queue=QueueNames.NOTIFY)
 
     current_app.logger.info("Sent notification to queue")
@@ -614,6 +636,11 @@ def send_user_reset_password():
     )
     saved_notification.personalisation = personalisation
 
+    redis_store.set(
+        f"email-personalisation-{saved_notification.id}",
+        json.dumps(personalisation),
+        ex=60 * 60,
+    )
     send_notification_to_queue(saved_notification, queue=QueueNames.NOTIFY)
 
     return jsonify({}), 204

migrations/versions/0410_enums_for_everything.py

@@ -468,6 +468,19 @@ def upgrade():
             existing_nullable=False,
             postgresql_using=enum_using("notification_type", NotificationType),
         )
+        # Clobbering bad data here. These are values we don't use any more, and anything with them is unnecessary.
+        op.execute("""
+            delete from
+                service_permissions
+            where
+                permission in (
+                    'letter',
+                    'letters_as_pdf',
+                    'upload_letters',
+                    'international_letters',
+                    'broadcast'
+                );
+        """)
         op.alter_column(
             "service_permissions",
             "permission",

terraform/demo/main.tf

@@ -1,38 +1,45 @@
 locals {
-  cf_org_name      = "gsa-tts-benefits-studio"
+  cf_org_name              = "gsa-tts-benefits-studio"
-  cf_space_name    = "notify-demo"
+  cf_space_name            = "notify-demo"
-  env              = "demo"
+  env                      = "demo"
-  app_name         = "notify-api"
+  app_name                 = "notify-api"
-  recursive_delete = false
+  delete_recursive_allowed = false
+}
+
+data "cloudfoundry_org" "org" {
+  name = local.cf_org_name
+}
+
+resource "cloudfoundry_space" "notify-demo" {
+  delete_recursive_allowed = local.delete_recursive_allowed
+  name                     = local.cf_space_name
+  org                      = data.cloudfoundry_org.org.id
 }
 
 module "database" {
   source = "github.com/18f/terraform-cloudgov//database?ref=v0.7.1"
 
-  cf_org_name      = local.cf_org_name
+  cf_org_name   = local.cf_org_name
-  cf_space_name    = local.cf_space_name
+  cf_space_name = local.cf_space_name
-  name             = "${local.app_name}-rds-${local.env}"
+  name          = "${local.app_name}-rds-${local.env}"
-  recursive_delete = local.recursive_delete
+  rds_plan_name = "micro-psql"
-  rds_plan_name    = "micro-psql"
 }
 
 module "redis" {
   source = "github.com/18f/terraform-cloudgov//redis?ref=v0.7.1"
 
-  cf_org_name      = local.cf_org_name
+  cf_org_name     = local.cf_org_name
-  cf_space_name    = local.cf_space_name
+  cf_space_name   = local.cf_space_name
-  name             = "${local.app_name}-redis-${local.env}"
+  name            = "${local.app_name}-redis-${local.env}"
-  recursive_delete = local.recursive_delete
+  redis_plan_name = "redis-dev"
-  redis_plan_name  = "redis-dev"
 }
 
 module "csv_upload_bucket" {
   source = "github.com/18f/terraform-cloudgov//s3?ref=v0.7.1"
 
-  cf_org_name      = local.cf_org_name
+  cf_org_name   = local.cf_org_name
-  cf_space_name    = local.cf_space_name
+  cf_space_name = local.cf_space_name
-  recursive_delete = local.recursive_delete
+  name          = "${local.app_name}-csv-upload-bucket-${local.env}"
-  name             = "${local.app_name}-csv-upload-bucket-${local.env}"
 }
 
 module "egress-space" {
@@ -40,6 +47,7 @@ module "egress-space" {
 
   cf_org_name              = local.cf_org_name
   cf_restricted_space_name = local.cf_space_name
+  delete_recursive_allowed = local.delete_recursive_allowed
   deployers = [
     var.cf_user,
     "steven.reilly@gsa.gov"
@@ -52,7 +60,6 @@ module "ses_email" {
   cf_org_name         = local.cf_org_name
   cf_space_name       = local.cf_space_name
   name                = "${local.app_name}-ses-${local.env}"
-  recursive_delete    = local.recursive_delete
   aws_region          = "us-west-2"
   email_domain        = "notify.sandbox.10x.gsa.gov"
   email_receipt_error = "notify-support@gsa.gov"
@@ -64,7 +71,6 @@ module "sns_sms" {
   cf_org_name         = local.cf_org_name
   cf_space_name       = local.cf_space_name
   name                = "${local.app_name}-sns-${local.env}"
-  recursive_delete    = local.recursive_delete
   aws_region          = "us-east-1"
   monthly_spend_limit = 25
 }

terraform/demo/providers.tf

@@ -3,7 +3,7 @@ terraform {
   required_providers {
     cloudfoundry = {
       source  = "cloudfoundry-community/cloudfoundry"
-      version = "0.53.0"
+      version = "0.53.1"
     }
   }
 

terraform/production/main.tf

@@ -1,45 +1,56 @@
 locals {
-  cf_org_name      = "gsa-tts-benefits-studio"
+  cf_org_name              = "gsa-tts-benefits-studio"
-  cf_space_name    = "notify-production"
+  cf_space_name            = "notify-production"
-  env              = "production"
+  env                      = "production"
-  app_name         = "notify-api"
+  app_name                 = "notify-api"
-  recursive_delete = false
+  delete_recursive_allowed = false
+  allow_ssh                = false
+}
+
+data "cloudfoundry_org" "org" {
+  name = local.cf_org_name
+}
+
+resource "cloudfoundry_space" "notify-production" {
+  allow_ssh                = local.allow_ssh
+  delete_recursive_allowed = local.delete_recursive_allowed
+  name                     = local.cf_space_name
+  org                      = data.cloudfoundry_org.org.id
 }
 
 module "database" {
   source = "github.com/18f/terraform-cloudgov//database?ref=v0.7.1"
 
-  cf_org_name      = local.cf_org_name
+  cf_org_name   = local.cf_org_name
-  cf_space_name    = local.cf_space_name
+  cf_space_name = local.cf_space_name
-  name             = "${local.app_name}-rds-${local.env}"
+  name          = "${local.app_name}-rds-${local.env}"
-  recursive_delete = local.recursive_delete
+  rds_plan_name = "small-psql-redundant"
-  rds_plan_name    = "small-psql-redundant"
 }
 
 module "redis" {
   source = "github.com/18f/terraform-cloudgov//redis?ref=v0.7.1"
 
-  cf_org_name      = local.cf_org_name
+  cf_org_name     = local.cf_org_name
-  cf_space_name    = local.cf_space_name
+  cf_space_name   = local.cf_space_name
-  name             = "${local.app_name}-redis-${local.env}"
+  name            = "${local.app_name}-redis-${local.env}"
-  recursive_delete = local.recursive_delete
+  redis_plan_name = "redis-3node-large"
-  redis_plan_name  = "redis-3node-large"
 }
 
 module "csv_upload_bucket" {
   source = "github.com/18f/terraform-cloudgov//s3?ref=v0.7.1"
 
-  cf_org_name      = local.cf_org_name
+  cf_org_name   = local.cf_org_name
-  cf_space_name    = local.cf_space_name
+  cf_space_name = local.cf_space_name
-  recursive_delete = local.recursive_delete
+  name          = "${local.app_name}-csv-upload-bucket-${local.env}"
-  name             = "${local.app_name}-csv-upload-bucket-${local.env}"
 }
 
 module "egress-space" {
   source = "../shared/egress_space"
 
+  allow_ssh                = local.allow_ssh
   cf_org_name              = local.cf_org_name
   cf_restricted_space_name = local.cf_space_name
+  delete_recursive_allowed = local.delete_recursive_allowed
   deployers = [
     var.cf_user
   ]
@@ -51,7 +62,6 @@ module "ses_email" {
   cf_org_name         = local.cf_org_name
   cf_space_name       = local.cf_space_name
   name                = "${local.app_name}-ses-${local.env}"
-  recursive_delete    = local.recursive_delete
   aws_region          = "us-gov-west-1"
   email_domain        = "notify.gov"
   mail_from_subdomain = "mail"
@@ -64,7 +74,6 @@ module "sns_sms" {
   cf_org_name         = local.cf_org_name
   cf_space_name       = local.cf_space_name
   name                = "${local.app_name}-sns-${local.env}"
-  recursive_delete    = local.recursive_delete
   aws_region          = "us-gov-west-1"
   monthly_spend_limit = 1000
 }

terraform/production/providers.tf

@@ -3,7 +3,7 @@ terraform {
   required_providers {
     cloudfoundry = {
       source  = "cloudfoundry-community/cloudfoundry"
-      version = "0.53.0"
+      version = "0.53.1"
     }
   }
 

terraform/shared/egress_space/main.tf

@@ -11,7 +11,8 @@ data "cloudfoundry_org" "org" {
 ###
 
 resource "cloudfoundry_space" "public_egress" {
-  delete_recursive_allowed = false
+  allow_ssh                = var.allow_ssh
+  delete_recursive_allowed = var.delete_recursive_allowed
   name                     = "${var.cf_restricted_space_name}-egress"
   org                      = data.cloudfoundry_org.org.id
 }

terraform/shared/egress_space/variables.tf

@@ -3,3 +3,15 @@ variable "cf_restricted_space_name" {}
 variable "deployers" {
   type = set(string)
 }
+
+variable "delete_recursive_allowed" {
+  type        = bool
+  default     = true
+  description = "Flag for allowing resources to be recursively deleted - not recommended in production environments"
+}
+
+variable "allow_ssh" {
+  type        = bool
+  default     = true
+  description = "Flag for allowing SSH access in a space - not recommended in production environments"
+}