Merge pull request #174 from GSA/ses-brokerpak

Provision SES via brokerpak

app/__init__.py

@@ -95,9 +95,8 @@ def create_app(application):
     logging.init_app(application)
     aws_sns_client.init_app(application, statsd_client=statsd_client)
 
-    aws_ses_client.init_app(application.config['AWS_REGION'], statsd_client=statsd_client)
+    aws_ses_client.init_app(statsd_client=statsd_client)
     aws_ses_stub_client.init_app(
-        application.config['AWS_REGION'],
         statsd_client=statsd_client,
         stub_url=application.config['SES_STUB_URL']
     )

app/clients/email/aws_ses.py

@@ -1,7 +1,7 @@
 from time import monotonic
 
-import boto3
 import botocore
+from boto3 import client
 from flask import current_app
 
 from app.clients import STATISTICS_DELIVERED, STATISTICS_FAILURE
@@ -10,6 +10,7 @@ from app.clients.email import (
     EmailClientException,
     EmailClientNonRetryableException,
 )
+from app.cloudfoundry_config import cloud_config
 
 ses_response_map = {
     'Permanent': {
@@ -56,8 +57,13 @@ class AwsSesClient(EmailClient):
     Amazon SES email client.
     '''
 
-    def init_app(self, region, statsd_client, *args, **kwargs):
-        self._client = boto3.client('ses', region_name=region)
+    def init_app(self, statsd_client, *args, **kwargs):
+        self._client = client(
+            'ses',
+            region_name=cloud_config.ses_region,
+            aws_access_key_id=cloud_config.ses_access_key,
+            aws_secret_access_key=cloud_config.ses_secret_key
+        )
         super(AwsSesClient, self).__init__(*args, **kwargs)
         self.statsd_client = statsd_client
 

app/clients/email/aws_ses_stub.py

@@ -12,7 +12,7 @@ class AwsSesStubClientException(EmailClientException):
 
 
 class AwsSesStubClient(EmailClient):
-    def init_app(self, region, statsd_client, stub_url):
+    def init_app(self, statsd_client, stub_url):
         self.statsd_client = statsd_client
         self.url = stub_url
 

app/cloudfoundry_config.py

@@ -31,5 +31,47 @@ class CloudfoundryConfig:
     def s3_credentials(self, service_name):
         return self.s3_buckets.get(service_name) or self._empty_bucket_credentials
 
+    @property
+    def ses_email_domain(self):
+        try:
+            return self._ses_credentials('domain_arn').split('/')[-1]
+        except KeyError:
+            return getenv('NOTIFY_EMAIL_DOMAIN', 'notify.sandbox.10x.gsa.gov')
+
+    @property
+    def ses_region(self):
+        try:
+            return self._ses_credentials('region')
+        except KeyError:
+            return getenv('AWS_REGION')
+
+    @property
+    def ses_access_key(self):
+        try:
+            return self._ses_credentials('smtp_user')
+        except KeyError:
+            return getenv('AWS_ACCESS_KEY_ID')
+
+    @property
+    def ses_secret_key(self):
+        try:
+            return self._ses_credentials('secret_access_key')
+        except KeyError:
+            return getenv('AWS_SECRET_ACCESS_KEY')
+
+    @property
+    def sns_topic_arns(self):
+        try:
+            return [
+                self._ses_credentials('bounce_topic_arn'),
+                self._ses_credentials('complaint_topic_arn'),
+                self._ses_credentials('delivery_topic_arn')
+            ]
+        except KeyError:
+            return []
+
+    def _ses_credentials(self, key):
+        return self.parsed_services['datagov-smtp'][0]['credentials'][key]
+
 
 cloud_config = CloudfoundryConfig()

app/config.py

@@ -108,11 +108,11 @@ class Config(object):
     AWS_US_TOLL_FREE_NUMBER = getenv("AWS_US_TOLL_FREE_NUMBER")
     # Whether to ignore POSTs from SNS for replies to SMS we sent
     RECEIVE_INBOUND_SMS = False
-    NOTIFY_EMAIL_DOMAIN = getenv('NOTIFY_EMAIL_DOMAIN', 'notify.sandbox.10x.gsa.gov')
+    NOTIFY_EMAIL_DOMAIN = cloud_config.ses_email_domain
     SES_STUB_URL = None  # TODO: set to a URL in env and remove this to use a stubbed SES service
     # AWS SNS topics for delivery receipts
     VALIDATE_SNS_TOPICS = True
-    VALID_SNS_TOPICS = ['notify_test_bounce', 'notify_test_success', 'notify_test_complaint', 'notify_test_sms_inbound']
+    VALID_SNS_TOPICS = cloud_config.sns_topic_arns
 
     # these should always add up to 100%
     SMS_PROVIDER_RESTING_POINTS = {

app/notifications/sns_cert_validator.py

@@ -39,8 +39,7 @@ def get_certificate(url):
 def validate_arn(sns_payload):
     if VALIDATE_SNS_TOPICS:
         arn = sns_payload.get('TopicArn')
-        topic_name = arn.split(':')[5]
-        if topic_name not in VALID_SNS_TOPICS:
+        if arn not in VALID_SNS_TOPICS:
             raise ValidationError("Invalid Topic Name")
 
 

docs/infra-overview.md

@@ -6,6 +6,34 @@ Notify is a Flask application running on [cloud.gov](https://cloud.gov), which a
 
 In addition to the Flask app, Notify uses Celery to manage the task queue. Celery stores tasks in Redis.
 
+## GitHub Repositories
+
+Application, infrastructure, and compliance work is spread across several repositories:
+
+### Application
+
+* [notifications-api](https://github.com/GSA/notifications-api) for the API app
+* [notifications-admin](https://github.com/GSA/notifications-admin) for the Admin UI app
+* [notifications-utils](https://github.com/GSA/notifications-utils) for common library functions
+
+### Infrastructure
+
+In addition to terraform directories in the api and admin apps above:
+
+#### We maintain:
+
+* [usnotify-ssb](https://github.com/GSA/usnotify-ssb) A supplemental service broker that provisions SES and SNS for us
+* [ttsnotify-brokerpak-sms](https://github.com/GSA/ttsnotify-brokerpak-sms) The brokerpak defining SNS (SMS sending)
+
+#### We use:
+
+* [datagov-brokerpak-smtp](https://github.com/GSA-TTS/datagov-brokerpak-smtp) The brokerpak defining SES
+* [cg-egress-proxy](https://github.com/GSA-TTS/cg-egress-proxy/) The caddy proxy that allows external API calls
+
+### Compliance
+
+* [us-notify-compliance](https://github.com/GSA/us-notify-compliance) for OSCAL control documentation and diagrams
+
 ## Terraform
 
 The cloud.gov environment is configured with Terraform. See [the `terraform` folder](../terraform/) to learn about that.
@@ -24,7 +52,7 @@ Through Pinpoint, the API needs at least one number so that the application itse
 
 The API also has access to AWS S3 buckets for storing CSVs of messages and contact lists. It does not access a third S3 bucket that stores agency logos.
 
-We may be able to provision these services through cloud.gov, as well. In addition to [s3 support](https://cloud.gov/docs/services/s3/), there is [an SES brokerpak](https://github.com/GSA-TTS/datagov-brokerpak-smtp) and work on an SNS brokerpak.
+SES and SNS for use by the cloud.gov-deployed apps is currently in the process of migrating to being provisioned through cloud.gov. Currently, SES, SNS, and S3 for local-development are still manually provisioned in AWS.
 
 ## New Relic
 
@@ -44,8 +72,11 @@ We are using [New Relic](https://one.newrelic.com/nr1-core?account=3389907) for
 
 ### Steps to prepare SES
 
-1. Go to SES console for \$AWS_REGION and create new origin and destination emails. AWS will send a verification via email which you'll need to complete.
-2. Find and replace instances in the repo of "testsender", "testreceiver" and "dispostable.com", with your origin and destination email addresses, which you verified in step 1 above.
+1. After the first deploy of the application with the SSB-brokered SES service completes:
+    1. Log into the SES console and navigate to the SNS subscription page.
+    2. Select "Request confirmation" for any subscriptions still in "Pending Confirmation" state
+2. (For sandbox SES accounts) Go to SES console for \$AWS_REGION and create new origin and destination emails. AWS will send a verification via email which you'll need to complete.
+3. Find and replace instances in the repo of "testsender", "testreceiver" and "dispostable.com", with your origin and destination email addresses, which you verified in step 1 above.
 
 TODO: create env vars for these origin and destination email addresses for the root service, and create new migrations to update postgres seed fixtures
 

manifest.yml

@@ -13,6 +13,9 @@ applications:
       - notify-api-redis-((env))
       - notify-api-csv-upload-bucket-((env))
       - notify-api-contact-list-bucket-((env))
+      - name: notify-api-ses-((env))
+        parameters:
+          notification_webhook: "https://((public_api_route))/notifications/email/ses"
 
     processes:
       - type: web

terraform/demo/main.tf

@@ -55,3 +55,15 @@ module "egress-space" {
     "steven.reilly@gsa.gov"
   ]
 }
+
+module "ses_email" {
+  source = "../shared/ses"
+
+  cf_org_name         = local.cf_org_name
+  cf_space_name       = local.cf_space_name
+  name                = "${local.app_name}-ses-${local.env}"
+  recursive_delete    = local.recursive_delete
+  aws_region          = "us-gov-west-1"
+  email_domain        = "notify.sandbox.10x.gsa.gov"
+  email_receipt_error = "notify-support@gsa.gov"
+}

terraform/production/main.tf

@@ -54,6 +54,18 @@ module "egress-space" {
   ]
 }
 
+module "ses_email" {
+  source = "../shared/ses"
+
+  cf_org_name         = local.cf_org_name
+  cf_space_name       = local.cf_space_name
+  name                = "${local.app_name}-ses-${local.env}"
+  recursive_delete    = local.recursive_delete
+  aws_region          = "us-gov-west-1"
+  email_domain        = "notify.gov"
+  email_receipt_error = "notify-support@gsa.gov"
+}
+
 ###########################################################################
 # The following lines need to be commented out for the initial `terraform apply`
 # It can be re-enabled after:

terraform/sandbox/main.tf

@@ -55,3 +55,14 @@ module "egress-space" {
     "steven.reilly@gsa.gov"
   ]
 }
+
+module "ses_email" {
+  source = "../shared/ses"
+
+  cf_org_name         = local.cf_org_name
+  cf_space_name       = local.cf_space_name
+  name                = "${local.app_name}-ses-${local.env}"
+  recursive_delete    = local.recursive_delete
+  aws_region          = "us-west-2"
+  email_receipt_error = "notify-support@gsa.gov"
+}

terraform/shared/ses/main.tf

@@ -0,0 +1,29 @@
+###
+# Target space/org
+###
+
+data "cloudfoundry_space" "space" {
+  org_name = var.cf_org_name
+  name     = var.cf_space_name
+}
+
+###
+# SES instance
+###
+
+data "cloudfoundry_service" "ses" {
+  name = "datagov-smtp"
+}
+
+resource "cloudfoundry_service_instance" "ses" {
+  name             = var.name
+  space            = data.cloudfoundry_space.space.id
+  service_plan     = data.cloudfoundry_service.ses.service_plans["base"]
+  recursive_delete = var.recursive_delete
+  json_params = jsonencode({
+    region                        = var.aws_region
+    domain                        = var.email_domain
+    email_receipt_error           = var.email_receipt_error
+    enable_feedback_notifications = true
+  })
+}

terraform/shared/ses/providers.tf

@@ -0,0 +1,9 @@
+terraform {
+  required_version = "~> 1.0"
+  required_providers {
+    cloudfoundry = {
+      source  = "cloudfoundry-community/cloudfoundry"
+      version = "~> 0.15"
+    }
+  }
+}

terraform/shared/ses/variables.tf

@@ -0,0 +1,36 @@
+variable "cf_org_name" {
+  type        = string
+  description = "cloud.gov organization name"
+}
+
+variable "cf_space_name" {
+  type        = string
+  description = "cloud.gov space name (staging or prod)"
+}
+
+variable "name" {
+  type        = string
+  description = "name of the service instance"
+}
+
+variable "recursive_delete" {
+  type        = bool
+  description = "when true, deletes service bindings attached to the resource (not recommended for production)"
+  default     = false
+}
+
+variable "aws_region" {
+  type        = string
+  description = "AWS region the SES instance is in"
+}
+
+variable "email_domain" {
+  type        = string
+  default     = ""
+  description = "domain name that emails will be coming from"
+}
+
+variable "email_receipt_error" {
+  type        = string
+  description = "email address to list in SPF records for errors to be sent to"
+}

terraform/staging/main.tf

@@ -55,3 +55,14 @@ module "egress-space" {
     "steven.reilly@gsa.gov"
   ]
 }
+
+module "ses_email" {
+  source = "../shared/ses"
+
+  cf_org_name         = local.cf_org_name
+  cf_space_name       = local.cf_space_name
+  name                = "${local.app_name}-ses-${local.env}"
+  recursive_delete    = local.recursive_delete
+  aws_region          = "us-west-2"
+  email_receipt_error = "notify-support@gsa.gov"
+}