diff --git a/app/__init__.py b/app/__init__.py index 456194694..38df2f5f7 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -95,9 +95,8 @@ def create_app(application): logging.init_app(application) aws_sns_client.init_app(application, statsd_client=statsd_client) - aws_ses_client.init_app(application.config['AWS_REGION'], statsd_client=statsd_client) + aws_ses_client.init_app(statsd_client=statsd_client) aws_ses_stub_client.init_app( - application.config['AWS_REGION'], statsd_client=statsd_client, stub_url=application.config['SES_STUB_URL'] ) diff --git a/app/clients/email/aws_ses.py b/app/clients/email/aws_ses.py index 1f8b2f75c..59d2243ac 100644 --- a/app/clients/email/aws_ses.py +++ b/app/clients/email/aws_ses.py @@ -1,7 +1,7 @@ from time import monotonic -import boto3 import botocore +from boto3 import client from flask import current_app from app.clients import STATISTICS_DELIVERED, STATISTICS_FAILURE @@ -10,6 +10,7 @@ from app.clients.email import ( EmailClientException, EmailClientNonRetryableException, ) +from app.cloudfoundry_config import cloud_config ses_response_map = { 'Permanent': { @@ -56,8 +57,13 @@ class AwsSesClient(EmailClient): Amazon SES email client. ''' - def init_app(self, region, statsd_client, *args, **kwargs): - self._client = boto3.client('ses', region_name=region) + def init_app(self, statsd_client, *args, **kwargs): + self._client = client( + 'ses', + region_name=cloud_config.ses_region, + aws_access_key_id=cloud_config.ses_access_key, + aws_secret_access_key=cloud_config.ses_secret_key + ) super(AwsSesClient, self).__init__(*args, **kwargs) self.statsd_client = statsd_client diff --git a/app/clients/email/aws_ses_stub.py b/app/clients/email/aws_ses_stub.py index bce9d3e9a..ef5fc8c13 100644 --- a/app/clients/email/aws_ses_stub.py +++ b/app/clients/email/aws_ses_stub.py @@ -12,7 +12,7 @@ class AwsSesStubClientException(EmailClientException): class AwsSesStubClient(EmailClient): - def init_app(self, region, statsd_client, stub_url): + def init_app(self, statsd_client, stub_url): self.statsd_client = statsd_client self.url = stub_url diff --git a/app/cloudfoundry_config.py b/app/cloudfoundry_config.py index aa7b3b543..a23ee260c 100644 --- a/app/cloudfoundry_config.py +++ b/app/cloudfoundry_config.py @@ -31,5 +31,47 @@ class CloudfoundryConfig: def s3_credentials(self, service_name): return self.s3_buckets.get(service_name) or self._empty_bucket_credentials + @property + def ses_email_domain(self): + try: + return self._ses_credentials('domain_arn').split('/')[-1] + except KeyError: + return getenv('NOTIFY_EMAIL_DOMAIN', 'notify.sandbox.10x.gsa.gov') + + @property + def ses_region(self): + try: + return self._ses_credentials('region') + except KeyError: + return getenv('AWS_REGION') + + @property + def ses_access_key(self): + try: + return self._ses_credentials('smtp_user') + except KeyError: + return getenv('AWS_ACCESS_KEY_ID') + + @property + def ses_secret_key(self): + try: + return self._ses_credentials('secret_access_key') + except KeyError: + return getenv('AWS_SECRET_ACCESS_KEY') + + @property + def sns_topic_arns(self): + try: + return [ + self._ses_credentials('bounce_topic_arn'), + self._ses_credentials('complaint_topic_arn'), + self._ses_credentials('delivery_topic_arn') + ] + except KeyError: + return [] + + def _ses_credentials(self, key): + return self.parsed_services['datagov-smtp'][0]['credentials'][key] + cloud_config = CloudfoundryConfig() diff --git a/app/config.py b/app/config.py index 039da6a43..d0201086f 100644 --- a/app/config.py +++ b/app/config.py @@ -108,11 +108,11 @@ class Config(object): AWS_US_TOLL_FREE_NUMBER = getenv("AWS_US_TOLL_FREE_NUMBER") # Whether to ignore POSTs from SNS for replies to SMS we sent RECEIVE_INBOUND_SMS = False - NOTIFY_EMAIL_DOMAIN = getenv('NOTIFY_EMAIL_DOMAIN', 'notify.sandbox.10x.gsa.gov') + NOTIFY_EMAIL_DOMAIN = cloud_config.ses_email_domain SES_STUB_URL = None # TODO: set to a URL in env and remove this to use a stubbed SES service # AWS SNS topics for delivery receipts VALIDATE_SNS_TOPICS = True - VALID_SNS_TOPICS = ['notify_test_bounce', 'notify_test_success', 'notify_test_complaint', 'notify_test_sms_inbound'] + VALID_SNS_TOPICS = cloud_config.sns_topic_arns # these should always add up to 100% SMS_PROVIDER_RESTING_POINTS = { diff --git a/app/notifications/sns_cert_validator.py b/app/notifications/sns_cert_validator.py index 1fbddd811..1b3f7ea3d 100644 --- a/app/notifications/sns_cert_validator.py +++ b/app/notifications/sns_cert_validator.py @@ -39,8 +39,7 @@ def get_certificate(url): def validate_arn(sns_payload): if VALIDATE_SNS_TOPICS: arn = sns_payload.get('TopicArn') - topic_name = arn.split(':')[5] - if topic_name not in VALID_SNS_TOPICS: + if arn not in VALID_SNS_TOPICS: raise ValidationError("Invalid Topic Name") diff --git a/docs/infra-overview.md b/docs/infra-overview.md index 301d618d4..09ed4df97 100644 --- a/docs/infra-overview.md +++ b/docs/infra-overview.md @@ -6,6 +6,34 @@ Notify is a Flask application running on [cloud.gov](https://cloud.gov), which a In addition to the Flask app, Notify uses Celery to manage the task queue. Celery stores tasks in Redis. +## GitHub Repositories + +Application, infrastructure, and compliance work is spread across several repositories: + +### Application + +* [notifications-api](https://github.com/GSA/notifications-api) for the API app +* [notifications-admin](https://github.com/GSA/notifications-admin) for the Admin UI app +* [notifications-utils](https://github.com/GSA/notifications-utils) for common library functions + +### Infrastructure + +In addition to terraform directories in the api and admin apps above: + +#### We maintain: + +* [usnotify-ssb](https://github.com/GSA/usnotify-ssb) A supplemental service broker that provisions SES and SNS for us +* [ttsnotify-brokerpak-sms](https://github.com/GSA/ttsnotify-brokerpak-sms) The brokerpak defining SNS (SMS sending) + +#### We use: + +* [datagov-brokerpak-smtp](https://github.com/GSA-TTS/datagov-brokerpak-smtp) The brokerpak defining SES +* [cg-egress-proxy](https://github.com/GSA-TTS/cg-egress-proxy/) The caddy proxy that allows external API calls + +### Compliance + +* [us-notify-compliance](https://github.com/GSA/us-notify-compliance) for OSCAL control documentation and diagrams + ## Terraform The cloud.gov environment is configured with Terraform. See [the `terraform` folder](../terraform/) to learn about that. @@ -24,7 +52,7 @@ Through Pinpoint, the API needs at least one number so that the application itse The API also has access to AWS S3 buckets for storing CSVs of messages and contact lists. It does not access a third S3 bucket that stores agency logos. -We may be able to provision these services through cloud.gov, as well. In addition to [s3 support](https://cloud.gov/docs/services/s3/), there is [an SES brokerpak](https://github.com/GSA-TTS/datagov-brokerpak-smtp) and work on an SNS brokerpak. +SES and SNS for use by the cloud.gov-deployed apps is currently in the process of migrating to being provisioned through cloud.gov. Currently, SES, SNS, and S3 for local-development are still manually provisioned in AWS. ## New Relic @@ -44,8 +72,11 @@ We are using [New Relic](https://one.newrelic.com/nr1-core?account=3389907) for ### Steps to prepare SES -1. Go to SES console for \$AWS_REGION and create new origin and destination emails. AWS will send a verification via email which you'll need to complete. -2. Find and replace instances in the repo of "testsender", "testreceiver" and "dispostable.com", with your origin and destination email addresses, which you verified in step 1 above. +1. After the first deploy of the application with the SSB-brokered SES service completes: + 1. Log into the SES console and navigate to the SNS subscription page. + 2. Select "Request confirmation" for any subscriptions still in "Pending Confirmation" state +2. (For sandbox SES accounts) Go to SES console for \$AWS_REGION and create new origin and destination emails. AWS will send a verification via email which you'll need to complete. +3. Find and replace instances in the repo of "testsender", "testreceiver" and "dispostable.com", with your origin and destination email addresses, which you verified in step 1 above. TODO: create env vars for these origin and destination email addresses for the root service, and create new migrations to update postgres seed fixtures diff --git a/manifest.yml b/manifest.yml index ae64daf6c..abf8dde6f 100644 --- a/manifest.yml +++ b/manifest.yml @@ -13,6 +13,9 @@ applications: - notify-api-redis-((env)) - notify-api-csv-upload-bucket-((env)) - notify-api-contact-list-bucket-((env)) + - name: notify-api-ses-((env)) + parameters: + notification_webhook: "https://((public_api_route))/notifications/email/ses" processes: - type: web diff --git a/terraform/demo/main.tf b/terraform/demo/main.tf index 86ce9ba53..3b7144a6a 100644 --- a/terraform/demo/main.tf +++ b/terraform/demo/main.tf @@ -55,3 +55,15 @@ module "egress-space" { "steven.reilly@gsa.gov" ] } + +module "ses_email" { + source = "../shared/ses" + + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + name = "${local.app_name}-ses-${local.env}" + recursive_delete = local.recursive_delete + aws_region = "us-gov-west-1" + email_domain = "notify.sandbox.10x.gsa.gov" + email_receipt_error = "notify-support@gsa.gov" +} diff --git a/terraform/production/main.tf b/terraform/production/main.tf index 2bcae487b..42bfff731 100644 --- a/terraform/production/main.tf +++ b/terraform/production/main.tf @@ -54,6 +54,18 @@ module "egress-space" { ] } +module "ses_email" { + source = "../shared/ses" + + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + name = "${local.app_name}-ses-${local.env}" + recursive_delete = local.recursive_delete + aws_region = "us-gov-west-1" + email_domain = "notify.gov" + email_receipt_error = "notify-support@gsa.gov" +} + ########################################################################### # The following lines need to be commented out for the initial `terraform apply` # It can be re-enabled after: diff --git a/terraform/sandbox/main.tf b/terraform/sandbox/main.tf index 6e8bb11f2..156ba57d3 100644 --- a/terraform/sandbox/main.tf +++ b/terraform/sandbox/main.tf @@ -55,3 +55,14 @@ module "egress-space" { "steven.reilly@gsa.gov" ] } + +module "ses_email" { + source = "../shared/ses" + + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + name = "${local.app_name}-ses-${local.env}" + recursive_delete = local.recursive_delete + aws_region = "us-west-2" + email_receipt_error = "notify-support@gsa.gov" +} diff --git a/terraform/shared/ses/main.tf b/terraform/shared/ses/main.tf new file mode 100644 index 000000000..de7ca2a2b --- /dev/null +++ b/terraform/shared/ses/main.tf @@ -0,0 +1,29 @@ +### +# Target space/org +### + +data "cloudfoundry_space" "space" { + org_name = var.cf_org_name + name = var.cf_space_name +} + +### +# SES instance +### + +data "cloudfoundry_service" "ses" { + name = "datagov-smtp" +} + +resource "cloudfoundry_service_instance" "ses" { + name = var.name + space = data.cloudfoundry_space.space.id + service_plan = data.cloudfoundry_service.ses.service_plans["base"] + recursive_delete = var.recursive_delete + json_params = jsonencode({ + region = var.aws_region + domain = var.email_domain + email_receipt_error = var.email_receipt_error + enable_feedback_notifications = true + }) +} diff --git a/terraform/shared/ses/providers.tf b/terraform/shared/ses/providers.tf new file mode 100644 index 000000000..8db86ca90 --- /dev/null +++ b/terraform/shared/ses/providers.tf @@ -0,0 +1,9 @@ +terraform { + required_version = "~> 1.0" + required_providers { + cloudfoundry = { + source = "cloudfoundry-community/cloudfoundry" + version = "~> 0.15" + } + } +} diff --git a/terraform/shared/ses/variables.tf b/terraform/shared/ses/variables.tf new file mode 100644 index 000000000..c56468cc6 --- /dev/null +++ b/terraform/shared/ses/variables.tf @@ -0,0 +1,36 @@ +variable "cf_org_name" { + type = string + description = "cloud.gov organization name" +} + +variable "cf_space_name" { + type = string + description = "cloud.gov space name (staging or prod)" +} + +variable "name" { + type = string + description = "name of the service instance" +} + +variable "recursive_delete" { + type = bool + description = "when true, deletes service bindings attached to the resource (not recommended for production)" + default = false +} + +variable "aws_region" { + type = string + description = "AWS region the SES instance is in" +} + +variable "email_domain" { + type = string + default = "" + description = "domain name that emails will be coming from" +} + +variable "email_receipt_error" { + type = string + description = "email address to list in SPF records for errors to be sent to" +} diff --git a/terraform/staging/main.tf b/terraform/staging/main.tf index a2ea6d0d8..174ea7b38 100644 --- a/terraform/staging/main.tf +++ b/terraform/staging/main.tf @@ -55,3 +55,14 @@ module "egress-space" { "steven.reilly@gsa.gov" ] } + +module "ses_email" { + source = "../shared/ses" + + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + name = "${local.app_name}-ses-${local.env}" + recursive_delete = local.recursive_delete + aws_region = "us-west-2" + email_receipt_error = "notify-support@gsa.gov" +}