Merge pull request #185 from GSA/main

Demo deploy

.github/actions/deploy-proxy/action.yml

@@ -9,7 +9,7 @@ inputs:
     required: true
   proxy_repo:
     description: git repo for cg-egress-proxy
-    default: https://github.com/GSA/cg-egress-proxy.git
+    default: https://github.com/GSA-TTS/cg-egress-proxy.git
   proxy_version:
     description: git ref to be deployed
     default: main

.github/workflows/checks.yml

@@ -8,8 +8,9 @@ permissions:
 env:
   DEBUG: True
   NOTIFY_ENVIRONMENT: test
+  NEW_RELIC_CONFIG_FILE: newrelic.ini
+  NEW_RELIC_ENVIRONMENT: test
   FLASK_APP: application.py
-  FLASK_ENV: development
   WERKZEUG_DEBUG_PIN: off
   REDIS_ENABLED: 0
   AWS_REGION: us-west-2
@@ -52,6 +53,21 @@ jobs:
       env:
         SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api
 
+  validate-new-relic-config:
+    runs-on: ubuntu-latest
+    environment: staging
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/setup-project
+      - name: Install pipenv packages
+        run: pipenv install --dev
+      - name: Validate NewRelic config
+        env:
+          NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
+          # Need to set a NEW_RELIC_ENVIRONMENT with monitor_mode: true
+          NEW_RELIC_ENVIRONMENT: staging
+        run: pipenv run newrelic-admin validate-config $NEW_RELIC_CONFIG_FILE
+
   pip-audit:
     runs-on: ubuntu-latest
     steps:

.github/workflows/daily_checks.yml

@@ -12,8 +12,9 @@ permissions:
 env:
   DEBUG: True
   NOTIFY_ENVIRONMENT: test
+  NEW_RELIC_CONFIG_FILE: newrelic.ini
+  NEW_RELIC_ENVIRONMENT: test
   FLASK_APP: application.py
-  FLASK_ENV: development
   WERKZEUG_DEBUG_PIN: off
   NOTIFY_EMAIL_DOMAIN: dispostable.com
   REDIS_ENABLED: 0

.github/workflows/deploy-demo.yml

@@ -53,6 +53,7 @@ jobs:
         ADMIN_CLIENT_SECRET: ${{ secrets.ADMIN_CLIENT_SECRET }}
         AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
         AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
       with:
         cf_username: ${{ secrets.CLOUDGOV_USERNAME }}
         cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
@@ -65,6 +66,7 @@ jobs:
           --var ADMIN_CLIENT_SECRET="$ADMIN_CLIENT_SECRET"
           --var AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID"
           --var AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY"
+          --var NEW_RELIC_LICENSE_KEY="$NEW_RELIC_LICENSE_KEY"
 
     - name: Check for changes to egress config
       id: changed-egress-config

.github/workflows/deploy.yml

@@ -58,6 +58,7 @@ jobs:
         ADMIN_CLIENT_SECRET: ${{ secrets.ADMIN_CLIENT_SECRET }}
         AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
         AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
       with:
         cf_username: ${{ secrets.CLOUDGOV_USERNAME }}
         cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
@@ -70,6 +71,7 @@ jobs:
           --var ADMIN_CLIENT_SECRET="$ADMIN_CLIENT_SECRET"
           --var AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID"
           --var AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY"
+          --var NEW_RELIC_LICENSE_KEY="$NEW_RELIC_LICENSE_KEY"
 
     - name: Check for changes to egress config
       id: changed-egress-config

.profile

@@ -4,3 +4,4 @@
 ##
 
 export https_proxy=$egress_proxy
+export NEW_RELIC_PROXY_HOST=$egress_proxy

Makefile

@@ -26,12 +26,12 @@ run-procfile:
 
 .PHONY: run-flask
 run-flask: ## Run flask
-	pipenv run flask run -p 6011 --host=0.0.0.0
+	pipenv run newrelic-admin run-program flask run -p 6011 --host=0.0.0.0
 
 .PHONY: run-celery
 run-celery: ## Run celery, TODO remove purge for staging/prod
 	pipenv run celery -A run_celery.notify_celery purge -f
-	pipenv run celery \
+	pipenv run newrelic-admin run-program celery \
 		-A run_celery.notify_celery worker \
 		--pidfile="/tmp/celery.pid" \
 		--loglevel=INFO \
@@ -52,6 +52,7 @@ generate-version-file: ## Generates the app version file
 	@echo -e "__git_commit__ = \"${GIT_COMMIT}\"\n__time__ = \"${DATE}\"" > ${APP_VERSION_FILE}
 
 .PHONY: test
+test: export NEW_RELIC_ENVIRONMENT=test
 test: ## Run tests
 	pipenv run flake8 .
 	pipenv run isort --check-only ./app ./tests

Pipfile

@@ -35,7 +35,7 @@ dnspython = "==2.2.1"
 docopt = "==0.6.2"
 docutils = "==0.16"
 eventlet = "==0.33.1"
-flask = "~=2.1.2"
+flask = "~=2.2"
 flask-bcrypt = "==1.0.1"
 flask-marshmallow = "==0.14.0"
 flask-migrate = "==3.1.0"
@@ -54,12 +54,13 @@ psycopg2-binary = "==2.9.3"
 pyjwt = "==2.4.0"
 python-dotenv = "==0.20.0"
 sqlalchemy = "==1.4.40"
-werkzeug = "~=2.1.1"
+werkzeug = "~=2.2"
 # gds metrics packages
 prometheus-client = "==0.14.1"
 gds-metrics = {version = "==0.2.4", ref = "6f1840a57b6fb1ee40b7e84f2f18ec229de8aa72", git = "https://github.com/alphagov/gds_metrics_python.git"}
 packaging = "==21.3"
 notifications-utils = {editable = true, ref = "main", git = "https://github.com/GSA/notifications-utils.git"}
+newrelic = "*"
 
 [dev-packages]
 flake8 = "==4.0.1"

app/__init__.py

@@ -95,9 +95,8 @@ def create_app(application):
     logging.init_app(application)
     aws_sns_client.init_app(application, statsd_client=statsd_client)
 
-    aws_ses_client.init_app(application.config['AWS_REGION'], statsd_client=statsd_client)
+    aws_ses_client.init_app(statsd_client=statsd_client)
     aws_ses_stub_client.init_app(
-        application.config['AWS_REGION'],
         statsd_client=statsd_client,
         stub_url=application.config['SES_STUB_URL']
     )

app/clients/email/aws_ses.py

@@ -1,7 +1,7 @@
 from time import monotonic
 
-import boto3
 import botocore
+from boto3 import client
 from flask import current_app
 
 from app.clients import STATISTICS_DELIVERED, STATISTICS_FAILURE
@@ -10,6 +10,7 @@ from app.clients.email import (
     EmailClientException,
     EmailClientNonRetryableException,
 )
+from app.cloudfoundry_config import cloud_config
 
 ses_response_map = {
     'Permanent': {
@@ -56,8 +57,13 @@ class AwsSesClient(EmailClient):
     Amazon SES email client.
     '''
 
-    def init_app(self, region, statsd_client, *args, **kwargs):
+    def init_app(self, statsd_client, *args, **kwargs):
-        self._client = boto3.client('ses', region_name=region)
+        self._client = client(
+            'ses',
+            region_name=cloud_config.ses_region,
+            aws_access_key_id=cloud_config.ses_access_key,
+            aws_secret_access_key=cloud_config.ses_secret_key
+        )
         super(AwsSesClient, self).__init__(*args, **kwargs)
         self.statsd_client = statsd_client
 

app/clients/email/aws_ses_stub.py

@@ -12,7 +12,7 @@ class AwsSesStubClientException(EmailClientException):
 
 
 class AwsSesStubClient(EmailClient):
-    def init_app(self, region, statsd_client, stub_url):
+    def init_app(self, statsd_client, stub_url):
         self.statsd_client = statsd_client
         self.url = stub_url
 

app/cloudfoundry_config.py

@@ -31,5 +31,47 @@ class CloudfoundryConfig:
     def s3_credentials(self, service_name):
         return self.s3_buckets.get(service_name) or self._empty_bucket_credentials
 
+    @property
+    def ses_email_domain(self):
+        try:
+            return self._ses_credentials('domain_arn').split('/')[-1]
+        except KeyError:
+            return getenv('NOTIFY_EMAIL_DOMAIN', 'notify.sandbox.10x.gsa.gov')
+
+    @property
+    def ses_region(self):
+        try:
+            return self._ses_credentials('region')
+        except KeyError:
+            return getenv('AWS_REGION')
+
+    @property
+    def ses_access_key(self):
+        try:
+            return self._ses_credentials('smtp_user')
+        except KeyError:
+            return getenv('AWS_ACCESS_KEY_ID')
+
+    @property
+    def ses_secret_key(self):
+        try:
+            return self._ses_credentials('secret_access_key')
+        except KeyError:
+            return getenv('AWS_SECRET_ACCESS_KEY')
+
+    @property
+    def sns_topic_arns(self):
+        try:
+            return [
+                self._ses_credentials('bounce_topic_arn'),
+                self._ses_credentials('complaint_topic_arn'),
+                self._ses_credentials('delivery_topic_arn')
+            ]
+        except KeyError:
+            return []
+
+    def _ses_credentials(self, key):
+        return self.parsed_services['datagov-smtp'][0]['credentials'][key]
+
 
 cloud_config = CloudfoundryConfig()

app/commands.py

@@ -65,6 +65,8 @@ from app.models import (
     Notification,
     Organisation,
     Service,
+    Template,
+    TemplateHistory,
     User,
 )
 from app.utils import get_local_midnight_in_utc
@@ -765,3 +767,28 @@ def create_user_jwt(token):
     service_id = token[-73:-37]
     api_key = token[-36:]
     print(create_jwt_token(api_key, service_id))
+
+
+def _update_template(id, name, template_type, content, subject):
+
+    template = Template.query.filter_by(id=id).first()
+    template.name = name
+    template.template_type = template_type
+    template.content = '\n'.join(content)
+    template.subject = subject
+
+    history = TemplateHistory.query.filter_by(id=id).first()
+    history.name = name
+    history.template_type = template_type
+    history.content = '\n'.join(content)
+    history.subject = subject
+
+    db.session.commit()
+
+
+@notify_command(name='update-templates')
+def update_templates():
+    with open(current_app.config['CONFIG_FILES'] + '/templates.json') as f:
+        data = json.load(f)
+        for d in data:
+            _update_template(d['id'], d['name'], d['type'], d['content'], d['subject'])

app/config.py

@@ -1,6 +1,6 @@
 import json
 from datetime import timedelta
-from os import getenv
+from os import getenv, path
 
 from celery.schedules import crontab
 from kombu import Exchange, Queue
@@ -108,11 +108,11 @@ class Config(object):
     AWS_US_TOLL_FREE_NUMBER = getenv("AWS_US_TOLL_FREE_NUMBER")
     # Whether to ignore POSTs from SNS for replies to SMS we sent
     RECEIVE_INBOUND_SMS = False
-    NOTIFY_EMAIL_DOMAIN = getenv('NOTIFY_EMAIL_DOMAIN', 'notify.sandbox.10x.gsa.gov')
+    NOTIFY_EMAIL_DOMAIN = cloud_config.ses_email_domain
     SES_STUB_URL = None  # TODO: set to a URL in env and remove this to use a stubbed SES service
     # AWS SNS topics for delivery receipts
     VALIDATE_SNS_TOPICS = True
-    VALID_SNS_TOPICS = ['notify_test_bounce', 'notify_test_success', 'notify_test_complaint', 'notify_test_sms_inbound']
+    VALID_SNS_TOPICS = cloud_config.sns_topic_arns
 
     # these should always add up to 100%
     SMS_PROVIDER_RESTING_POINTS = {
@@ -147,6 +147,9 @@ class Config(object):
     MAX_LETTER_PDF_ZIP_FILESIZE = 40 * 1024 * 1024  # 40mb
     MAX_LETTER_PDF_COUNT_PER_ZIP = 500
 
+    # Default data
+    CONFIG_FILES = path.dirname(__file__) + '/config_files/'
+
     NOTIFY_SERVICE_ID = 'd6aa2c68-a2d9-4437-ab19-3ae8eb202553'
     NOTIFY_USER_ID = '6af522d0-2915-4e52-83a3-3690455a5fe6'
     INVITATION_EMAIL_TEMPLATE_ID = '4f46df42-f795-4cc4-83bb-65ca312f49cc'

app/config_files/templates.json

@@ -0,0 +1,274 @@
+[
+    {
+        "id": "42a23d19-504e-49bb-a95e-4976baff4757",
+        "name": "Example text message template",
+        "type": "sms",
+        "subject":  "",
+        "content": ["Hi, I’m trying out U.S. Notify. Today is ((day of week)) and my favorite color is ((color))."]
+    },
+    {
+        "id": "4f46df42-f795-4cc4-83bb-65ca312f49cc",
+        "name": "Notify invitation email",
+        "type": "email",
+        "subject":  "((user_name)) has invited you to collaborate on ((service_name)) on U.S. Notify",
+        "content": ["((user_name)) has invited you to collaborate on ((service_name)) on U.S. Notify.",
+            "",
+            "",
+            "U.S. Notify makes it easy to keep people updated by helping you send text messages, emails and letters.",
+            "",
+            "",
+            "Click this link to create an account on U.S. Notify:",
+            "",
+            "((url))",
+            "",
+            "",
+            "This invitation will stop working at midnight tomorrow. This is to keep ((service_name)) secure."]
+    },
+    {
+        "id": "36fb0730-6259-4da1-8a80-c8de22ad4246",
+        "name": "Notify SMS verify code",
+        "type": "sms",
+        "subject": "",
+        "content": ["((verify_code)) is your U.S. Notify authentication code"]
+    },
+    {
+        "id": "474e9242-823b-4f99-813d-ed392e7f1201",
+        "name": "Notify password reset email",
+        "type": "email",
+        "subject": "Reset your U.S. Notify password",
+        "content": ["Hi ((user_name)),",
+            "",
+            "",
+            "We received a request to reset your password on U.S. Notify.",
+            "",
+            "",
+            "If you didn’t request this email, you can ignore it – your password has not been changed.",
+            "",
+            "",
+            "To reset your password, click this link:","","","((url))"]
+    },
+    {
+        "id": "299726d2-dba6-42b8-8209-30e1d66ea164",
+        "name": "Notify email verify code",
+        "type": "email",
+        "subject": "Sign in to U.S. Notify",
+        "content": ["Hi ((name)),",
+        "",
+        "",
+        "To sign in to U.S. Notify please open this link:",
+        "",
+        "",
+        "((url))"]
+    },
+    {
+        "id": "ece42649-22a8-4d06-b87f-d52d5d3f0a27",
+        "name": "Notify email verification code",
+        "type": "email",
+        "subject": "Confirm U.S. Notify registration",
+        "content": ["Hi ((name)),",
+        "",
+        "",
+        "To complete your registration for U.S. Notify please click the link below",
+        "",
+        "",
+        "((url))"]
+    },
+    {
+        "id": "0880fbb1-a0c6-46f0-9a8e-36c986381ceb",
+        "name": "Your U.S. Notify account",
+        "type": "email",
+        "subject": "Your U.S. Notify account",
+        "content": ["You already have a U.S. Notify account with this email address.",
+        "",
+        "",
+        "Sign in here: ((signin_url))",
+        "",
+        "",
+        "If you’ve forgotten your password, you can reset it here: ((forgot_password_url))",
+        "",
+        "",
+        "",
+        "",
+        "If you didn’t try to register for a U.S. Notify account recently, please let us know here: ((feedback_url))"]
+    },
+    {
+        "id": "eb4d9930-87ab-4aef-9bce-786762687884",
+        "name": "Confirm new email address",
+        "type": "email",
+        "subject": "Confirm your email address for U.S. Notify",
+        "content": ["Hi ((name)),","","","Click this link to confirm your new email address:",
+            "",
+            "",
+            "((url))",
+            "",
+            "",
+            "If you didn’t try to change the email address for your U.S. Notify account, let us know here:",
+            "",
+            "",
+            "((feedback_url))"]
+    },
+    {
+        "id": "618185c6-3636-49cd-b7d2-6f6f5eb3bdde",
+        "name": "Automated \"You’re now live\" message",
+        "type": "email",
+        "subject": "((service name)) is now live on U.S. Notify",
+        "content": ["Hi ((name)),",
+        "",
+        "",
+        "((service name)) is now live on U.S. Notify.",
+        "",
+        "",
+        "You can send up to ((message limit)) messages per day.",
+        "",
+        "",
+        "As a live service, you’ll need to know who to contact if you have a question, or something goes wrong.",
+        "",
+        "",
+        "If our system status page shows a problem, then we’ve been alerted and are working on it – you don’t need to contact us.",
+        "",
+        "",
+        "#Problems or questions out of hours",
+        "",
+        "",
+        "We offer out of hours support for emergencies.",
+        "",
+        "",
+        "It’s only an emergency if:",
+        "",
+        "*   no one in your team can log in",
+        "",
+        "*   a ‘technical difficulties’ error appears when you try to upload a file",
+        "",
+        "*   a 500 response code appears when you try to send messages using the API",
+        "",
+        "",
+        "If you have one of these emergencies, email details to:",
+        "",
+        "notify-support@gsa.gov",
+        "",
+        "",
+        "^Only use this email address for out of hours emergencies. Don’t share this address with people outside of your team.",
+        "",
+        "",
+        "We’ll get back to you within 30 minutes and give you hourly updates until the problem’s fixed.",
+        "",
+        "",
+        "For non-emergency problems or questions, use our support page and we’ll reply in office hours.",
+        "",
+        "",
+        "Thanks",
+        "",
+        "U.S. Notify team"]
+    },
+    {
+        "id": "203566f0-d835-47c5-aa06-932439c86573",
+        "name": "Notify organization invitation email",
+        "type": "email",
+        "subject": "((user_name)) has invited you to collaborate on ((organisation_name)) on U.S. Notify",
+        "content": ["((user_name)) has invited you to collaborate on ((organisation_name)) on U.S. Notify.","","","U.S. Notify makes it easy to keep people updated by helping you send text messages, emails and letters.","","","Open this link to create an account on U.S. Notify:","","((url))","","","This invitation will stop working at midnight tomorrow. This is to keep ((organisation_name)) secure."]
+    },
+    {
+        "id": "c73f1d71-4049-46d5-a647-d013bdeca3f0",
+        "name": "Email address changed by service manager",
+        "type": "email",
+        "subject": "Your U.S. Notify email address has changed",
+        "content":  ["Dear ((name)),","","","((servicemanagername)) changed your Notify account email address to:","","","((email address))","","","You’ll need to use this email address next time you sign in.","","","Thanks","","","U.S. Notify team"]
+    },
+    {
+        "id": "8a31520f-4751-4789-8ea1-fe54496725eb",
+        "name": "Phone number changed by service manager",
+        "type": "sms",
+        "subject": "",
+        "content": ["Your mobile number was changed by ((servicemanagername)). Next time you sign in, your U.S. Notify authentication code will be sent to this phone."]
+    },
+    {
+        "id": "a42f1d17-9404-46d5-a647-d013bdfca3e1",
+        "name": "Verify email reply-to address for a service",
+        "type": "email",
+        "subject": "Your U.S. Notify reply-to email address",
+        "content": ["Hi,","","","This address has been provided as a reply-to email address for a U.S. Notify account.","","Any replies from users to emails they receive through U.S. Notify will come back to this email address.","","","This is just a quick check to make sure the address is valid.","","","No need to reply.","","","Thanks","","","U.S. Notify team"]
+    },
+    {
+        "id": "4fd2e43c-309b-4e50-8fb8-1955852d9d71",
+        "name": "MOU Signed By Receipt",
+        "type": "email",
+        "subject": "You’ve accepted the U.S. Notify data sharing and financial agreement",
+        "content": [
+            "Hi ((signed_by_name)),",
+            "",
+            "((org_name)) has accepted the U.S. Notify data sharing and financial agreement. ",
+            "",
+            "If you need another copy of the agreement you can download it here: ((mou_link))",
+            "",
+            "",
+            "Thanks,",
+            "U.S. Notify team"
+        ]
+    },
+    {
+        "id": "c20206d5-bf03-4002-9a90-37d5032d9e84",
+        "name": "MOU Signed On Behalf Of Receipt - Signed by",
+        "type": "email",
+        "subject": "You’ve accepted the U.S. Notify data sharing and financial agreement",
+        "content": [
+            "Hi ((signed_by_name)),",
+            "",
+            "((org_name)) has accepted the U.S. Notify data sharing and financial agreement. We’ve emailed ((on_behalf_of_name)) to let them know too.",
+            "",
+            "If you need another copy of the agreement you can download it here: ((mou_link))",
+            "",
+            "",
+            "Thanks,",
+            "U.S. Notify team"
+        ]
+    },
+    {
+        "id": "522b6657-5ca5-4368-a294-6b527703bd0b",
+        "name": "MOU Signed On Behalf Of Receipt - On Behalf Of",
+        "type": "email",
+        "subject": "((org_name)) has accepted the U.S. Notify data sharing and financial agreement",
+        "content": [
+            "Hi ((on_behalf_of_name)),",
+            "",
+            "((signed_by_name)) has accepted the U.S. Notify data sharing and financial agreement on your behalf, for ((org_name)).",
+            "",
+            "U.S. Notify lets teams in the public sector send emails, text messages and letters. It’s built and run by a team in the TTS Public Benefits Studio (part of GSA).",
+            "",
+            "If you need another copy of the agreement you can download it here: ((mou_link))",
+            "",
+            "",
+            "Thanks,",
+            "U.S. Notify team"
+        ]
+    },
+    {
+        "id": "d0e66c4c-0c50-43f0-94f5-f85b613202d4",
+        "name": "MOU Signed Notify Team Alert",
+        "type": "email",
+        "subject": "Someone signed an MOU for an org on Notify",
+        "content": [
+            "What’s up Notifiers,",
+            "",
+            "((signed_by_name)) just accepted the data sharing and financial agreement for ((org_name)).",
+            "",
+            "See how ((org_name)) is using Notify here: ((org_dashboard_link))"
+        ]
+    },
+    {
+        "id": "11fad854-fd38-4a7c-bd17-805fb13dfc12",
+        "name": "Notify daily letter volumes",
+        "type": "email",
+        "subject": "Notify letter volume for ((date)): ((total_volume)) letters, ((total_sheets)) sheets",
+        "content": [
+            "((total_volume)) letters (((total_sheets)) sheets) sent via Notify are coming in today’s batch. These include: ",
+            "",
+            "((first_class_volume)) first class letters (((first_class_sheets)) sheets).",
+            "((second_class_volume)) second class letters (((second_class_sheets)) sheets).",
+            "((international_volume)) international letters (((international_sheets)) sheets).",
+            "",
+            "Thanks",
+            "",
+            "U.S. Notify team"
+        ]
+    }
+]

app/dao/users_dao.py

@@ -1,6 +1,6 @@
 import uuid
 from datetime import datetime, timedelta
-from random import SystemRandom
+from secrets import randbelow
 
 from sqlalchemy import func
 from sqlalchemy.orm import joinedload
@@ -19,16 +19,9 @@ def _remove_values_for_keys_if_present(dict, keys):
         dict.pop(key, None)
 
 
-def create_secret_code():
+def create_secret_code(length=6):
-    return ''.join(get_non_repeating_random_digits(5))
+    random_number = randbelow(10 ** length)
-
+    return "{:0{length}d}".format(random_number, length=length)
-
-def get_non_repeating_random_digits(length):
-    output = [None] * length
-    for index in range(length):
-        while output[index] in {None, output[index - 1]}:
-            output[index] = str(SystemRandom().randrange(10))
-    return output
 
 
 def save_user_attribute(usr, update_dict=None):

app/notifications/process_notifications.py

@@ -105,13 +105,13 @@ def persist_notification(
     document_download_count=None,
     updated_at=None
 ):
-    current_app.logger.info('Presisting notification')
+    current_app.logger.info('Persisting notification')
 
     notification_created_at = created_at or datetime.utcnow()
     if not notification_id:
         notification_id = uuid.uuid4()
 
-    current_app.logger.info('Presisting notification with id {}'.format(notification_id))
+    current_app.logger.info('Persisting notification with id {}'.format(notification_id))
 
     notification = Notification(
         id=notification_id,
@@ -136,7 +136,7 @@ def persist_notification(
         updated_at=updated_at
     )
 
-    current_app.logger.info('Presisting notification with to address: {}'.format(notification.to))
+    current_app.logger.info('Persisting notification with to address: {}'.format(notification.to))
 
     if notification_type == SMS_TYPE:
         formatted_recipient = validate_and_format_phone_number(recipient, international=True)
@@ -146,9 +146,9 @@ def persist_notification(
         notification.phone_prefix = recipient_info.country_prefix
         notification.rate_multiplier = recipient_info.billable_units
     elif notification_type == EMAIL_TYPE:
-        current_app.logger.info('Presisting notification with type: {}'.format(EMAIL_TYPE))
+        current_app.logger.info('Persisting notification with type: {}'.format(EMAIL_TYPE))
         notification.normalised_to = format_email_address(notification.to)
-        current_app.logger.info('Presisting notification to formatted email: {}'.format(notification.normalised_to))
+        current_app.logger.info('Persisting notification to formatted email: {}'.format(notification.normalised_to))
     elif notification_type == LETTER_TYPE:
         notification.postage = postage
         notification.international = postage in INTERNATIONAL_POSTAGE_TYPES

app/notifications/sns_cert_validator.py

@@ -39,8 +39,7 @@ def get_certificate(url):
 def validate_arn(sns_payload):
     if VALIDATE_SNS_TOPICS:
         arn = sns_payload.get('TopicArn')
-        topic_name = arn.split(':')[5]
+        if arn not in VALID_SNS_TOPICS:
-        if topic_name not in VALID_SNS_TOPICS:
             raise ValidationError("Invalid Topic Name")
 
 

deploy-config/egress_proxy/notify-api-demo.allow.acl

@@ -1,2 +1,4 @@
 email.us-west-2.amazonaws.com
 sns.us-west-2.amazonaws.com
+gov-collector.newrelic.com
+egress-proxy-notify-api-demo.apps.internal

deploy-config/egress_proxy/notify-api-staging.allow.acl

@@ -1,2 +1,4 @@
 email.us-west-2.amazonaws.com
 sns.us-west-2.amazonaws.com
+gov-collector.newrelic.com
+egress-proxy-notify-api-staging.apps.internal

deploy-config/sandbox.yml

@@ -6,6 +6,6 @@ worker_memory: 512M
 scheduler_memory: 256M
 public_api_route: notify-api-sandbox.app.cloud.gov
 admin_base_url: https://notify-sandbox.app.cloud.gov
-ADMIN_CLIENT_SECRET: dev-notify-secret-key
+ADMIN_CLIENT_SECRET: sandbox-notify-secret-key
-DANGEROUS_SALT: dev-notify-salt
+DANGEROUS_SALT: sandbox-notify-salt
-SECRET_KEY: dev-notify-secret-key
+SECRET_KEY: sandbox-notify-secret-key

docs/infra-overview.md

@@ -6,6 +6,34 @@ Notify is a Flask application running on [cloud.gov](https://cloud.gov), which a
 
 In addition to the Flask app, Notify uses Celery to manage the task queue. Celery stores tasks in Redis.
 
+## GitHub Repositories
+
+Application, infrastructure, and compliance work is spread across several repositories:
+
+### Application
+
+* [notifications-api](https://github.com/GSA/notifications-api) for the API app
+* [notifications-admin](https://github.com/GSA/notifications-admin) for the Admin UI app
+* [notifications-utils](https://github.com/GSA/notifications-utils) for common library functions
+
+### Infrastructure
+
+In addition to terraform directories in the api and admin apps above:
+
+#### We maintain:
+
+* [usnotify-ssb](https://github.com/GSA/usnotify-ssb) A supplemental service broker that provisions SES and SNS for us
+* [ttsnotify-brokerpak-sms](https://github.com/GSA/ttsnotify-brokerpak-sms) The brokerpak defining SNS (SMS sending)
+
+#### We use:
+
+* [datagov-brokerpak-smtp](https://github.com/GSA-TTS/datagov-brokerpak-smtp) The brokerpak defining SES
+* [cg-egress-proxy](https://github.com/GSA-TTS/cg-egress-proxy/) The caddy proxy that allows external API calls
+
+### Compliance
+
+* [us-notify-compliance](https://github.com/GSA/us-notify-compliance) for OSCAL control documentation and diagrams
+
 ## Terraform
 
 The cloud.gov environment is configured with Terraform. See [the `terraform` folder](../terraform/) to learn about that.
@@ -22,9 +50,13 @@ In SNS, we have 3 topics for SMS receipts. These are not currently functional, s
 
 Through Pinpoint, the API needs at least one number so that the application itself can send SMS for authentication codes.
 
-The API also has access to AWS S3 buckets for storing CSVs of messages and contact lists. It does not access a third S3 bucket that stores agency logos. 
+The API also has access to AWS S3 buckets for storing CSVs of messages and contact lists. It does not access a third S3 bucket that stores agency logos.
 
-We may be able to provision these services through cloud.gov, as well. In addition to [s3 support](https://cloud.gov/docs/services/s3/), there is [an SES brokerpak](https://github.com/GSA-TTS/datagov-brokerpak-smtp) and work on an SNS brokerpak.
+SES and SNS for use by the cloud.gov-deployed apps is currently in the process of migrating to being provisioned through cloud.gov. Currently, SES, SNS, and S3 for local-development are still manually provisioned in AWS.
+
+## New Relic
+
+We are using [New Relic](https://one.newrelic.com/nr1-core?account=3389907) for application monitoring and error reporting. When requesting access to New Relic, ask to be added to the Benefits-Studio subaccount.
 
 ## Onboarding
 
@@ -32,6 +64,7 @@ We may be able to provision these services through cloud.gov, as well. In additi
 - [ ] Get permissions for the repos
 - [ ] Get access to the cloud.gov org && space
 - [ ] Get [access to AWS](https://handbook.tts.gsa.gov/launching-software/infrastructure/#cloud-service-provider-csp-sandbox-accounts), if necessary
+- [ ] Get [access to New Relic](https://handbook.tts.gsa.gov/tools/new-relic/#how-do-i-get-access-to-new-relic), if necessary
 - [ ] Pull down creds from cloud.gov and create the local .env file
 - [ ] Do stuff!
 
@@ -39,8 +72,11 @@ We may be able to provision these services through cloud.gov, as well. In additi
 
 ### Steps to prepare SES
 
-1. Go to SES console for \$AWS_REGION and create new origin and destination emails. AWS will send a verification via email which you'll need to complete.
+1. After the first deploy of the application with the SSB-brokered SES service completes:
-2. Find and replace instances in the repo of "testsender", "testreceiver" and "dispostable.com", with your origin and destination email addresses, which you verified in step 1 above.
+    1. Log into the SES console and navigate to the SNS subscription page.
+    2. Select "Request confirmation" for any subscriptions still in "Pending Confirmation" state
+2. (For sandbox SES accounts) Go to SES console for \$AWS_REGION and create new origin and destination emails. AWS will send a verification via email which you'll need to complete.
+3. Find and replace instances in the repo of "testsender", "testreceiver" and "dispostable.com", with your origin and destination email addresses, which you verified in step 1 above.
 
 TODO: create env vars for these origin and destination email addresses for the root service, and create new migrations to update postgres seed fixtures
 
@@ -54,4 +90,4 @@ TODO: create env vars for these origin and destination email addresses for the r
 6. Go to SNS console for \$AWS_PINPOINT_REGION, look at lefthand sidebar under "Mobile" and go to "Text Messaging (SMS)"
 7. Scroll down to "Sandbox destination phone numbers" and tap "Add phone number" then follow the steps to verify (you'll need to be able to retrieve a code sent to each number)
 
-At this point, you _should_ be able to complete both the email and phone verification steps of the Notify user sign up process! 🎉
+At this point, you _should_ be able to complete both the email and phone verification steps of the Notify user sign up process! 🎉

docs/system-description.md

@@ -0,0 +1,38 @@
+US Notify
+=========
+
+System Description
+------------------
+
+US Notify is a service being developed by the TTS Public Benefits Studio to increase the availability of
+SMS and email notifications to Federal, State, and Local Benefits agencies.
+
+Agencies that sign up will be able to create and use personalized message templates for sending
+notifications to members of the public regarding their benefits. These could include reminders
+about upcoming enrollment deadlines and tasks, or information about upcoming appointments, events,
+or services.
+
+The templates are sent by the agency using one of two methods:
+
+* using the US Notify API to send a message to a given recipient with given personalization values
+* using the US Notify website to upload a CSV file of recipients and their personalization values, one row per message
+
+### Environment
+
+US Notify is comprised of two applications both running on cloud.gov:
+
+* Admin, a Flask website running on the python_buildpack which hosts agency user-facing UI
+* API, a Flask application running on the python_buildpack hosting the US Notify API
+
+US Notify utilizes several cloud.gov-provided services:
+
+* S3 buckets for temporary file storage
+* Elasticache (redis) for cacheing data and enqueueing background tasks
+* RDS (PostgreSQL) for system data storage
+
+US Notify also provisions and uses two AWS services via a [supplemental service broker](https://github.com/GSA/datagov-ssb):
+
+* [SNS](https://aws.amazon.com/sns/) for sending SMS messages
+* [SES](https://aws.amazon.com/ses/) for sending email messages
+
+For further details of the system and how it connects to supporting services, see the [application boundary diagram](https://github.com/GSA/us-notify-compliance/blob/main/diagrams/rendered/apps/application.boundary.png)

manifest.yml

@@ -13,6 +13,9 @@ applications:
       - notify-api-redis-((env))
       - notify-api-csv-upload-bucket-((env))
       - notify-api-contact-list-bucket-((env))
+      - name: notify-api-ses-((env))
+        parameters:
+          notification_webhook: "https://((public_api_route))/notifications/email/ses"
 
     processes:
       - type: web
@@ -22,7 +25,7 @@ applications:
       - type: worker
         instances: ((worker_instances))
         memory: ((worker_memory))
-        command: celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4
+        command: newrelic-admin run-program celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4
       - type: scheduler
         instances: 1
         memory: ((scheduler_memory))
@@ -31,7 +34,10 @@ applications:
     env:
       NOTIFY_APP_NAME: api
       FLASK_APP: application.py
-      FLASK_ENV: production
+      FLASK_DEBUG: "false"
+      NEW_RELIC_CONFIG_FILE: newrelic.ini
+      NEW_RELIC_ENVIRONMENT: ((env))
+      NEW_RELIC_LICENSE_KEY: ((NEW_RELIC_LICENSE_KEY))
 
       NOTIFY_ENVIRONMENT: ((env))
       API_HOST_NAME: https://((public_api_route))
@@ -48,3 +54,4 @@ applications:
       AWS_US_TOLL_FREE_NUMBER: +18446120782
 
       REQUESTS_CA_BUNDLE: "/etc/ssl/certs/ca-certificates.crt"
+      NEW_RELIC_CA_BUNDLE_PATH: "/etc/ssl/certs/ca-certificates.crt"

migrations/alembic.ini

@@ -20,12 +20,12 @@ keys = console
 keys = generic
 
 [logger_root]
-level = WARN
+level = INFO
 handlers =
 qualname =
 
 [logger_sqlalchemy]
-level = WARN
+level = INFO
 handlers =
 qualname = sqlalchemy.engine
 

migrations/versions/0383_update_default_templates.py

@@ -0,0 +1,55 @@
+"""
+
+Revision ID: 0383_update_default_templates.py
+Revises: 0382_remove_old_providers
+Create Date: 2023-01-10 11:42:25.633265
+
+"""
+import json
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+from flask import current_app
+
+revision = '0383_update_default_templates.py'
+down_revision = '0382_remove_old_providers'
+
+
+def upgrade():
+    update = """
+        UPDATE {} SET name = '{}', template_type = '{}', content = '{}', subject = '{}'
+        WHERE id = '{}'
+    """
+
+    with open(current_app.config['CONFIG_FILES'] + '/templates.json') as f:
+        data = json.load(f)
+        for d in data:
+            for table_name in 'templates', 'templates_history':
+                op.execute(
+                    update.format(
+                        table_name,
+                        d['name'],
+                        d['type'],
+                        '\n'.join(d['content']),
+                        d.get('subject'),
+                        d['id']
+                    )
+                )
+
+            # op.execute(
+            #     """
+            #     INSERT INTO template_redacted
+            #     (
+            #         template_id,
+            #         redact_personalisation,
+            #         updated_at,
+            #         updated_by_id
+            #     ) VALUES ( '{}', false, current_timestamp, '{}' )
+            #     """.format(d['id'], current_app.config['NOTIFY_USER_ID'])
+            # )
+   
+
+def downgrade():
+    # with associated code changes, edits to templates should no longer be made via migration.
+    # instead, update the fixture and run the flask command to update.
+    pass

newrelic.ini

@@ -0,0 +1,222 @@
+# ---------------------------------------------------------------------------
+
+#
+# This file configures the New Relic Python Agent.
+#
+# The path to the configuration file should be supplied to the function
+# newrelic.agent.initialize() when the agent is being initialized.
+#
+# The configuration file follows a structure similar to what you would
+# find for Microsoft Windows INI files. For further information on the
+# configuration file format see the Python ConfigParser documentation at:
+#
+#    http://docs.python.org/library/configparser.html
+#
+# For further discussion on the behaviour of the Python agent that can
+# be configured via this configuration file see:
+#
+#    http://newrelic.com/docs/python/python-agent-configuration
+#
+
+# ---------------------------------------------------------------------------
+
+# Here are the settings that are common to all environments.
+
+[newrelic]
+
+# You must specify the license key associated with your New
+# Relic account. This key binds the Python Agent's data to your
+# account in the New Relic service.
+# license_key is set via NEW_RELIC_LICENSE_KEY env var
+
+host = gov-collector.newrelic.com
+
+# The application name. Set this to be the name of your
+# application as you would like it to show up in New Relic UI.
+# The UI will then auto-map instances of your application into a
+# entry on your home dashboard page.
+app_name = us-notify-api (Development)
+
+# New Relic offers distributed tracing for monitoring and analyzing modern
+# distributed systems.Enable distributed tracing.
+distributed_tracing.enabled = true
+
+# When "true", the agent collects performance data about your
+# application and reports this data to the New Relic UI at
+# newrelic.com. This global switch is normally overridden for
+# each environment below.
+monitor_mode = true
+
+# Sets the name of a file to log agent messages to. Useful for
+# debugging any issues with the agent. This is not set by
+# default as it is not known in advance what user your web
+# application processes will run as and where they have
+# permission to write to. Whatever you set this to you must
+# ensure that the permissions for the containing directory and
+# the file itself are correct, and that the user that your web
+# application runs as can write to the file. If not able to
+# write out a log file, it is also possible to say "stderr" and
+# output to standard error output. This would normally result in
+# output appearing in your web server log.
+#log_file = /tmp/newrelic-python-agent.log
+
+# Sets the level of detail of messages sent to the log file, if
+# a log file location has been provided. Possible values, in
+# increasing order of detail, are: "critical", "error", "warning",
+# "info" and "debug". When reporting any agent issues to New
+# Relic technical support, the most useful setting for the
+# support engineers is "debug". However, this can generate a lot
+# of information very quickly, so it is best not to keep the
+# agent at this level for longer than it takes to reproduce the
+# problem you are experiencing.
+log_level = info
+
+# The Python Agent communicates with the New Relic service using
+# SSL by default. Note that this does result in an increase in
+# CPU overhead, over and above what would occur for a non SSL
+# connection, to perform the encryption involved in the SSL
+# communication. This work is though done in a distinct thread
+# to those handling your web requests, so it should not impact
+# response times. You can if you wish revert to using a non SSL
+# connection, but this will result in information being sent
+# over a plain socket connection and will not be as secure.
+ssl = true
+
+# High Security Mode enforces certain security settings, and
+# prevents them from being overridden, so that no sensitive data
+# is sent to New Relic. Enabling High Security Mode means that
+# SSL is turned on, request parameters are not collected, and SQL
+# can not be sent to New Relic in its raw form. To activate High
+# Security Mode, it must be set to 'true' in this local .ini
+# configuration file AND be set to 'true' in the server-side
+# configuration in the New Relic user interface. For details, see
+# https://docs.newrelic.com/docs/subscriptions/high-security
+high_security = false
+
+# The Python Agent will attempt to connect directly to the New
+# Relic service. If there is an intermediate firewall between
+# your host and the New Relic service that requires you to use a
+# HTTP proxy, then you should set both the "proxy_host" and
+# "proxy_port" settings to the required values for the HTTP
+# proxy. The "proxy_user" and "proxy_pass" settings should
+# additionally be set if proxy authentication is implemented by
+# the HTTP proxy. The "proxy_scheme" setting dictates what
+# protocol scheme is used in talking to the HTTP proxy. This
+# would normally always be set as "http" which will result in the
+# agent then using a SSL tunnel through the HTTP proxy for end to
+# end encryption.
+# proxy_scheme = http
+# proxy_host = hostname
+# proxy_port = 8080
+# proxy_user =
+# proxy_pass =
+
+# Capturing request parameters is off by default. To enable the
+# capturing of request parameters, first ensure that the setting
+# "attributes.enabled" is set to "true" (the default value), and
+# then add "request.parameters.*" to the "attributes.include"
+# setting. For details about attributes configuration, please
+# consult the documentation.
+# attributes.include = request.parameters.*
+
+# The transaction tracer captures deep information about slow
+# transactions and sends this to the UI on a periodic basis. The
+# transaction tracer is enabled by default. Set this to "false"
+# to turn it off.
+transaction_tracer.enabled = true
+
+# Threshold in seconds for when to collect a transaction trace.
+# When the response time of a controller action exceeds this
+# threshold, a transaction trace will be recorded and sent to
+# the UI. Valid values are any positive float value, or (default)
+# "apdex_f", which will use the threshold for a dissatisfying
+# Apdex controller action - four times the Apdex T value.
+transaction_tracer.transaction_threshold = apdex_f
+
+# When the transaction tracer is on, SQL statements can
+# optionally be recorded. The recorder has three modes, "off"
+# which sends no SQL, "raw" which sends the SQL statement in its
+# original form, and "obfuscated", which strips out numeric and
+# string literals.
+transaction_tracer.record_sql = obfuscated
+
+# Threshold in seconds for when to collect stack trace for a SQL
+# call. In other words, when SQL statements exceed this
+# threshold, then capture and send to the UI the current stack
+# trace. This is helpful for pinpointing where long SQL calls
+# originate from in an application.
+transaction_tracer.stack_trace_threshold = 0.5
+
+# Determines whether the agent will capture query plans for slow
+# SQL queries. Only supported in MySQL and PostgreSQL. Set this
+# to "false" to turn it off.
+transaction_tracer.explain_enabled = true
+
+# Threshold for query execution time below which query plans
+# will not not be captured. Relevant only when "explain_enabled"
+# is true.
+transaction_tracer.explain_threshold = 0.5
+
+# Space separated list of function or method names in form
+# 'module:function' or 'module:class.function' for which
+# additional function timing instrumentation will be added.
+transaction_tracer.function_trace =
+
+# The error collector captures information about uncaught
+# exceptions or logged exceptions and sends them to UI for
+# viewing. The error collector is enabled by default. Set this
+# to "false" to turn it off.
+error_collector.enabled = true
+
+# To stop specific errors from reporting to the UI, set this to
+# a space separated list of the Python exception type names to
+# ignore. The exception name should be of the form 'module:class'.
+error_collector.ignore_classes =
+
+# Browser monitoring is the Real User Monitoring feature of the UI.
+# For those Python web frameworks that are supported, this
+# setting enables the auto-insertion of the browser monitoring
+# JavaScript fragments.
+browser_monitoring.auto_instrument = false
+
+# A thread profiling session can be scheduled via the UI when
+# this option is enabled. The thread profiler will periodically
+# capture a snapshot of the call stack for each active thread in
+# the application to construct a statistically representative
+# call tree.
+thread_profiler.enabled = true
+
+# ---------------------------------------------------------------------------
+
+#
+# The application environments. These are specific settings which
+# override the common environment settings. The settings related to a
+# specific environment will be used when the environment argument to the
+# newrelic.agent.initialize() function has been defined to be either
+# "development", "test", "staging" or "production".
+#
+
+[newrelic:development]
+app_name = us-notify-api (Development)
+
+[newrelic:test]
+app_name = us-notify-api (Test)
+monitor_mode = false
+
+[newrelic:sandbox]
+app_name = us-notify-api (Sandbox)
+monitor_mode = true
+
+[newrelic:staging]
+app_name = us-notify-api (Staging)
+monitor_mode = true
+
+[newrelic:demo]
+app_name = us-notify-api (Demo)
+monitor_mode = true
+
+[newrelic:production]
+app_name = us-notify-api
+monitor_mode = true
+
+# ---------------------------------------------------------------------------

sample.env

@@ -54,5 +54,11 @@ NOTIFY_APP_NAME=api
 
 # Flask
 FLASK_APP=application.py
-FLASK_ENV=development
+FLASK_DEBUG=true
 WERKZEUG_DEBUG_PIN=off
+
+#############################################################
+
+# New Relic
+NEW_RELIC_CONFIG_FILE=newrelic.ini
+NEW_RELIC_LICENSE_KEY="don't write secrets to the sample file"

scripts/migrate_and_run_web.sh

@@ -4,4 +4,4 @@ if [[ $CF_INSTANCE_INDEX -eq 0 ]]; then
   flask db upgrade
 fi
 
-exec gunicorn -c ${HOME}/gunicorn_config.py application
+exec newrelic-admin run-program gunicorn -c ${HOME}/gunicorn_config.py application

terraform/create_service_account.sh

@@ -18,9 +18,16 @@ Options:
 -o <ORG NAME>: configure the organization to act on. Default: $org
 
 Notes:
-OrgManager is required for terraform to create <env>-egress spaces
+* OrgManager is required for terraform to create <env>-egress spaces
+* Requires cf-cli@8
 "
 
+cf_version=`cf --version | cut -d " " -f 3`
+if [[ $cf_version != 8.* ]]; then
+  echo "$usage"
+  exit 1
+fi
+
 set -e
 set -o pipefail
 
@@ -67,7 +74,7 @@ cf create-service cloud-gov-service-account $role $service 1>&2
 cf create-service-key $service service-account-key 1>&2
 
 # output service key to stdout in secrets.auto.tfvars format
-creds=`cf service-key $service service-account-key | tail -n 4`
+creds=`cf service-key $service service-account-key | tail -n +2 | jq '.credentials'`
 username=`echo $creds | jq -r '.username'`
 password=`echo $creds | jq -r '.password'`
 

terraform/demo/main.tf

@@ -55,3 +55,26 @@ module "egress-space" {
     "steven.reilly@gsa.gov"
   ]
 }
+
+module "ses_email" {
+  source = "../shared/ses"
+
+  cf_org_name         = local.cf_org_name
+  cf_space_name       = local.cf_space_name
+  name                = "${local.app_name}-ses-${local.env}"
+  recursive_delete    = local.recursive_delete
+  aws_region          = "us-west-2"
+  email_domain        = "notify.sandbox.10x.gsa.gov"
+  email_receipt_error = "notify-support@gsa.gov"
+}
+
+module "sns_sms" {
+  source = "../shared/sns"
+
+  cf_org_name         = local.cf_org_name
+  cf_space_name       = local.cf_space_name
+  name                = "${local.app_name}-sns-${local.env}"
+  recursive_delete    = local.recursive_delete
+  aws_region          = "us-east-1"
+  monthly_spend_limit = 25
+}

terraform/production/main.tf

@@ -54,6 +54,33 @@ module "egress-space" {
   ]
 }
 
+module "ses_email" {
+  source = "../shared/ses"
+
+  cf_org_name         = local.cf_org_name
+  cf_space_name       = local.cf_space_name
+  name                = "${local.app_name}-ses-${local.env}"
+  recursive_delete    = local.recursive_delete
+  aws_region          = "us-gov-west-1"
+  email_domain        = "notify.gov"
+  email_receipt_error = "notify-support@gsa.gov"
+}
+
+#########################################################################
+# Wait for SNS is out of sandbox and spending limit is increased
+# before activating this module
+#########################################################################
+# module "sns_sms" {
+#   source = "../shared/sns"
+
+#   cf_org_name         = local.cf_org_name
+#   cf_space_name       = local.cf_space_name
+#   name                = "${local.app_name}-sns-${local.env}"
+#   recursive_delete    = local.recursive_delete
+#   aws_region          = "us-gov-west-1"
+#   monthly_spend_limit = 1000
+# }
+
 ###########################################################################
 # The following lines need to be commented out for the initial `terraform apply`
 # It can be re-enabled after:

terraform/sandbox/main.tf

@@ -55,3 +55,25 @@ module "egress-space" {
     "steven.reilly@gsa.gov"
   ]
 }
+
+module "ses_email" {
+  source = "../shared/ses"
+
+  cf_org_name         = local.cf_org_name
+  cf_space_name       = local.cf_space_name
+  name                = "${local.app_name}-ses-${local.env}"
+  recursive_delete    = local.recursive_delete
+  aws_region          = "us-west-2"
+  email_receipt_error = "notify-support@gsa.gov"
+}
+
+module "sns_sms" {
+  source = "../shared/sns"
+
+  cf_org_name         = local.cf_org_name
+  cf_space_name       = local.cf_space_name
+  name                = "${local.app_name}-sns-${local.env}"
+  recursive_delete    = local.recursive_delete
+  aws_region          = "us-east-2"
+  monthly_spend_limit = 1
+}

terraform/shared/ses/main.tf

@@ -0,0 +1,29 @@
+###
+# Target space/org
+###
+
+data "cloudfoundry_space" "space" {
+  org_name = var.cf_org_name
+  name     = var.cf_space_name
+}
+
+###
+# SES instance
+###
+
+data "cloudfoundry_service" "ses" {
+  name = "datagov-smtp"
+}
+
+resource "cloudfoundry_service_instance" "ses" {
+  name             = var.name
+  space            = data.cloudfoundry_space.space.id
+  service_plan     = data.cloudfoundry_service.ses.service_plans["base"]
+  recursive_delete = var.recursive_delete
+  json_params = jsonencode({
+    region                        = var.aws_region
+    domain                        = var.email_domain
+    email_receipt_error           = var.email_receipt_error
+    enable_feedback_notifications = true
+  })
+}

terraform/shared/ses/providers.tf

@@ -0,0 +1,9 @@
+terraform {
+  required_version = "~> 1.0"
+  required_providers {
+    cloudfoundry = {
+      source  = "cloudfoundry-community/cloudfoundry"
+      version = "~> 0.15"
+    }
+  }
+}

terraform/shared/ses/variables.tf

@@ -0,0 +1,36 @@
+variable "cf_org_name" {
+  type        = string
+  description = "cloud.gov organization name"
+}
+
+variable "cf_space_name" {
+  type        = string
+  description = "cloud.gov space name (staging or prod)"
+}
+
+variable "name" {
+  type        = string
+  description = "name of the service instance"
+}
+
+variable "recursive_delete" {
+  type        = bool
+  description = "when true, deletes service bindings attached to the resource (not recommended for production)"
+  default     = false
+}
+
+variable "aws_region" {
+  type        = string
+  description = "AWS region the SES instance is in"
+}
+
+variable "email_domain" {
+  type        = string
+  default     = ""
+  description = "domain name that emails will be coming from"
+}
+
+variable "email_receipt_error" {
+  type        = string
+  description = "email address to list in SPF records for errors to be sent to"
+}

terraform/shared/sns/main.tf

@@ -0,0 +1,27 @@
+###
+# Target space/org
+###
+
+data "cloudfoundry_space" "space" {
+  org_name = var.cf_org_name
+  name     = var.cf_space_name
+}
+
+###
+# SES instance
+###
+
+data "cloudfoundry_service" "sns" {
+  name = "ttsnotify-sms"
+}
+
+resource "cloudfoundry_service_instance" "sns" {
+  name             = var.name
+  space            = data.cloudfoundry_space.space.id
+  service_plan     = data.cloudfoundry_service.sns.service_plans["base"]
+  recursive_delete = var.recursive_delete
+  json_params = jsonencode({
+    region              = var.aws_region
+    monthly_spend_limit = var.monthly_spend_limit
+  })
+}

terraform/shared/sns/providers.tf

@@ -0,0 +1,9 @@
+terraform {
+  required_version = "~> 1.0"
+  required_providers {
+    cloudfoundry = {
+      source  = "cloudfoundry-community/cloudfoundry"
+      version = "~> 0.15"
+    }
+  }
+}

terraform/shared/sns/variables.tf

@@ -0,0 +1,30 @@
+variable "cf_org_name" {
+  type        = string
+  description = "cloud.gov organization name"
+}
+
+variable "cf_space_name" {
+  type        = string
+  description = "cloud.gov space name (staging or prod)"
+}
+
+variable "name" {
+  type        = string
+  description = "name of the service instance"
+}
+
+variable "recursive_delete" {
+  type        = bool
+  description = "when true, deletes service bindings attached to the resource (not recommended for production)"
+  default     = false
+}
+
+variable "aws_region" {
+  type        = string
+  description = "AWS region the SNS settings are set in"
+}
+
+variable "monthly_spend_limit" {
+  type        = number
+  description = "SMS budget limit in USD. Support request must be made before raising above 1"
+}

terraform/staging/main.tf

@@ -55,3 +55,25 @@ module "egress-space" {
     "steven.reilly@gsa.gov"
   ]
 }
+
+module "ses_email" {
+  source = "../shared/ses"
+
+  cf_org_name         = local.cf_org_name
+  cf_space_name       = local.cf_space_name
+  name                = "${local.app_name}-ses-${local.env}"
+  recursive_delete    = local.recursive_delete
+  aws_region          = "us-west-2"
+  email_receipt_error = "notify-support@gsa.gov"
+}
+
+module "sns_sms" {
+  source = "../shared/sns"
+
+  cf_org_name         = local.cf_org_name
+  cf_space_name       = local.cf_space_name
+  name                = "${local.app_name}-sns-${local.env}"
+  recursive_delete    = local.recursive_delete
+  aws_region          = "us-west-2"
+  monthly_spend_limit = 25
+}

tests/app/dao/test_users_dao.py

@@ -183,21 +183,15 @@ def test_create_secret_code_different_subsequent_codes():
     assert code1 != code2
 
 
-def test_create_secret_code_returns_5_digits():
+def test_create_secret_code_returns_6_digits():
     code = create_secret_code()
-    assert len(str(code)) == 5
+    assert len(code) == 6
 
 
-def test_create_secret_code_never_repeats_consecutive_digits(mocker):
+def test_create_secret_code_can_customize_digits():
-    mocker.patch('app.dao.users_dao.SystemRandom.randrange', side_effect=[
+    code_length = 10
-        1, 1, 1,
+    code = create_secret_code(code_length)
-        2,
+    assert len(code) == code_length
-        3,
-        4, 4,
-        1,  # Repeated allowed if not consecutive
-        9, 9,  # Not called because we have 5 digits now
-    ])
-    assert create_secret_code() == '12341'
 
 
 @freeze_time('2018-07-07 12:00:00')

tests/app/test_commands.py

@@ -1,12 +1,13 @@
 import pytest
 
 from app.commands import (
+    _update_template,
     create_test_user,
     insert_inbound_numbers_from_file,
     populate_annual_billing_with_defaults,
 )
 from app.dao.inbound_numbers_dao import dao_get_available_inbound_numbers
-from app.models import AnnualBilling, User
+from app.models import AnnualBilling, Template, User
 from tests.app.db import create_annual_billing, create_service
 
 
@@ -88,3 +89,20 @@ def test_populate_annual_billing_with_defaults_sets_free_allowance_to_zero_if_pr
 
     assert len(results) == 1
     assert results[0].free_sms_fragment_limit == 0
+
+
+def test_update_template(
+    notify_db_session, email_2fa_code_template
+):
+
+    _update_template(
+        "299726d2-dba6-42b8-8209-30e1d66ea164",
+        "Example text message template!",
+        "sms",
+        ["Hi, I’m trying out U.S. Notify! Today is ((day of week)) and my favorite color is ((color))."],
+        ""
+    )
+
+    t = Template.query.all()
+
+    assert t[0].name == "Example text message template!"