From f0cb13312927157d3d51133fc55cceefa11682ee Mon Sep 17 00:00:00 2001 From: James Moffet Date: Thu, 14 Jul 2022 13:36:05 -0500 Subject: [PATCH 1/2] update readme with SES/SNS instructions --- README.md | 57 ++++++++++++++++++++++++++++++------------------------ sample.env | 10 +++++----- 2 files changed, 37 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 73de43d2e..7f50a00bd 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # GOV.UK Notify API Contains: + - the public-facing REST API for GOV.UK Notify, which teams can integrate with using [our clients](https://www.notifications.service.gov.uk/documentation) - an internal-only REST API built using Flask to manage services, users, templates, etc (this is what the [admin app](http://github.com/alphagov/notifications-admin) talks to) - asynchronous workers built using Celery to put things on queues and read them off to be processed, sent to providers, updated, etc @@ -19,33 +20,37 @@ We run python 3.9 both locally and in production. To run the API you will need appropriate AWS credentials. See the [Wiki](https://github.com/alphagov/notifications-manuals/wiki/aws-accounts#how-to-set-up-local-development) for more details. -### `environment.sh` +### `.env` file -Creating and edit an environment.sh file. - -``` -echo " -export NOTIFY_ENVIRONMENT='development' - -export MMG_API_KEY='MMG_API_KEY' -export FIRETEXT_API_KEY='FIRETEXT_ACTUAL_KEY' -export NOTIFICATION_QUEUE_PREFIX='YOUR_OWN_PREFIX' - -export FLASK_APP=application.py -export FLASK_ENV=development -export WERKZEUG_DEBUG_PIN=off -"> environment.sh -``` +Create and edit a .env file, based on sample.env Things to change: -* Replace `YOUR_OWN_PREFIX` with `local_dev_`. -* Run the following in the credentials repo to get the API keys. +- Replace `YOUR_OWN_PREFIX` with `local_dev_` +- Replace `NOTIFY_EMAIL_DOMAIN` with the domain your emails will come from (i.e. the "origination email" in your SES project) +- Replace `SECRET_KEY` and `DANGEROUS_SALT` with high-entropy secret values +- Set up AWS SES and SNS as indicated in next section (AWS Setup), fill in missing AWS env vars -``` -notify-pass credentials/firetext -notify-pass credentials/mmg -``` +### AWS Setup + +**Steps to prepare SES** + +1. Go to SES console for \$AWS_REGION and create new origin and destination emails. AWS will send a verification via email which you'll need to complete. +2. Find and replace instances in the repo of "testsender", "testreceiver" and "dispostable.com", with your origin and destination email addresses, which you verified in step 1 above. + +TODO: create env vars for these origin and destination email addresses for the root service, and create new migrations to update postgres seed fixtures + +**Steps to prepare SNS** + +1. Go to Pinpoints console for \$AWS_PINPOINT_REGION and choose "create new project", then "configure for sms" +2. Tick the box at the top to enable SMS, choose "transactional" as the default type and save +3. In the lefthand sidebar, go the "SMS and Voice" (bottom) and choose "Phone Numbers" +4. Under "Number Settings" choose "Request Phone Number" +5. Choose Toll-free number, tick SMS, untick Voice, choose "transactional", hit next and then "request" +6. Go to SNS console for \$AWS_PINPOINT_REGION, look at lefthand sidebar under "Mobile" and go to "Text Messaging (SMS)" +7. Scroll down to "Sandbox destination phone numbers" and tap "Add phone number" then follow the steps to verify (you'll need to be able to retrieve a code sent to each number) + +At this point, you _should_ be able to complete both the email and phone verification steps of the Notify user sign up process! 🎉 ### Secrets Detection @@ -59,7 +64,7 @@ detect-secrets scan > .secrets.baseline Ideally, you'll install `detect-secrets` so that it's accessible from any environment from which you _might_ commit. You can use `brew install` to make it available globally. You could also install via `pip install` inside a virtual environment, if you're sure you'll _only_ commit from that environment. -If you open .git/hooks/pre-commit you should see a simple bash script that runs the command below, reads the output and aborts before committing if detect-secrets finds a secret. You should be able to test it by staging a file with any high-entropy string like `"bblfwk3u4bt484+afw4avev5ae+afr4?/fa"` (it also has other ways to detect secrets, this is just the most straightforward to test). +If you open .git/hooks/pre-commit you should see a simple bash script that runs the command below, reads the output and aborts before committing if detect-secrets finds a secret. You should be able to test it by staging a file with any high-entropy string like `"bblfwk3u4bt484+afw4avev5ae+afr4?/fa"` (it also has other ways to detect secrets, this is just the most straightforward to test). You can permit exceptions by adding an inline comment containing `pragma: allowlist secret` @@ -95,7 +100,7 @@ To use redis caching you need to switch it on with an environment variable: export REDIS_ENABLED=1 ``` -## To run the application +## To run the application ``` # install dependencies, etc. @@ -124,7 +129,7 @@ make run-celery-with-docker make run-celery-beat-with-docker ``` -## To test the application +## To test the application ``` # install dependencies, etc. @@ -140,11 +145,13 @@ care about: `flask db` contains alembic migration commands, and `flask command` example, to purge all dynamically generated functional test data, do the following: Locally + ``` flask command purge_functional_test_data -u ``` On the server + ``` cf run-task notify-api "flask command purge_functional_test_data -u " ``` diff --git a/sample.env b/sample.env index 9e9ccedd1..8d408e2e4 100644 --- a/sample.env +++ b/sample.env @@ -1,4 +1,4 @@ -### REBUILD THE DEVCONTAINER WHEN YOU MODIFY .ENV ### +# ## REBUILD THE DEVCONTAINER WHEN YOU MODIFY .ENV ### # Debug DEBUG=True @@ -38,7 +38,7 @@ SQLALCHEMY_DATABASE_TEST_URI=postgresql://postgres:chummy@db:5432/test_notificat # AWS AWS_REGION=us-west-2 -AWS_ACCESS_KEY_ID= -AWS_SECRET_ACCESS_KEY= -AWS_PINPOINT_REGION= -AWS_US_TOLL_FREE_NUMBER= \ No newline at end of file +AWS_ACCESS_KEY_ID=DO_NOT_ADD_SECRETS_TO_THIS_SAMPLE_FILE +AWS_SECRET_ACCESS_KEY=DO_NOT_ADD_SECRETS_TO_THIS_SAMPLE_FILE +AWS_PINPOINT_REGION=us-west-2 +AWS_US_TOLL_FREE_NUMBER=+18001111111 \ No newline at end of file From 18a04fd7199f0eb60662e8bf4f754baf3ec96901 Mon Sep 17 00:00:00 2001 From: James Moffet Date: Mon, 25 Jul 2022 15:19:05 -0700 Subject: [PATCH 2/2] configs --- Makefile | 10 ++-- README.md | 35 ++++++------- app/config.py | 81 +++++++++++++++-------------- devcontainer-api/.devcontainer.json | 15 +++--- manifest.yml | 2 - varsfile.sample | 4 -- 6 files changed, 71 insertions(+), 76 deletions(-) diff --git a/Makefile b/Makefile index c1aaeb999..0faec389f 100644 --- a/Makefile +++ b/Makefile @@ -146,16 +146,16 @@ cf-deploy: ## Deploys the app to Cloud Foundry cf-deploy-api-db-migration: $(if ${CF_SPACE},,$(error Must specify CF_SPACE)) cf target -o ${CF_ORG} -s ${CF_SPACE} - make -s CF_APP=notify-api-db-migration generate-manifest > ${CF_MANIFEST_PATH} + make -s CF_APP=notifications-api generate-manifest > ${CF_MANIFEST_PATH} - cf push notify-api-db-migration --no-route -f ${CF_MANIFEST_PATH} + cf push notifications-api --no-route -f ${CF_MANIFEST_PATH} rm ${CF_MANIFEST_PATH} - cf run-task notify-api-db-migration --command="flask db upgrade" --name api_db_migration + cf run-task notifications-api --command="flask db upgrade" --name api_db_migration .PHONY: cf-check-api-db-migration-task -cf-check-api-db-migration-task: ## Get the status for the last notify-api-db-migration task - @cf curl /v3/apps/`cf app --guid notify-api-db-migration`/tasks?order_by=-created_at | jq -r ".resources[0].state" +cf-check-api-db-migration-task: ## Get the status for the last notifications-api task + @cf curl /v3/apps/`cf app --guid notifications-api`/tasks?order_by=-created_at | jq -r ".resources[0].state" .PHONY: cf-rollback cf-rollback: ## Rollbacks the app to the previous release diff --git a/README.md b/README.md index 7f50a00bd..6ce7e77ac 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ The command that is actually run by the pre-commit hook is: `git diff --staged - You can also run against all tracked files staged or not: `git ls-files -z | xargs -0 detect-secrets-hook --baseline .secrets.baseline` -### Postgres +### Postgres [DEPRECATED] Install [Postgres.app](http://postgresapp.com/). @@ -84,7 +84,7 @@ Currently the API works with PostgreSQL 11. After installation, open the Postgre export PATH=${PATH}:/Applications/Postgres.app/Contents/Versions/11/bin/ ``` -### Redis +### Redis [DEPRECATED] To switch redis on you'll need to install it locally. On a Mac you can do: @@ -103,32 +103,27 @@ export REDIS_ENABLED=1 ## To run the application ``` -# install dependencies, etc. -make bootstrap +# set up AWS SES/SNS as instructed above -# run the web app +# create .env file as instructed above + +# download vscode and install the Remote-Containers plug-in from Microsoft + +# create the external docker network +docker network create notify-network + +# Using the command pallette, search "Remote Containers: Open folder in project" and choose devcontainer-api, then wait for docker to build + +# Open a terminal in vscode and run the web server make run-flask -# run the background tasks +# Open another terminal in vscode and run the background tasks make run-celery -# run scheduled tasks (optional) +# Open a third terminal in vscode and run scheduled tasks (optional) make run-celery-beat ``` -We've had problems running Celery locally due to one of its dependencies: pycurl. Due to the complexity of the issue, we also support running Celery via Docker: - -``` -# install dependencies, etc. -make bootstrap-with-docker - -# run the background tasks -make run-celery-with-docker - -# run scheduled tasks -make run-celery-beat-with-docker -``` - ## To test the application ``` diff --git a/app/config.py b/app/config.py index f5f7f9325..5b2fb1dca 100644 --- a/app/config.py +++ b/app/config.py @@ -82,44 +82,44 @@ class TaskNames(object): class Config(object): # URL of admin app - ADMIN_BASE_URL = os.getenv('ADMIN_BASE_URL') + ADMIN_BASE_URL = os.environ.get('ADMIN_BASE_URL') # URL of api app (on AWS this is the internal api endpoint) - API_HOST_NAME = os.getenv('API_HOST_NAME') + API_HOST_NAME = os.environ.get('API_HOST_NAME') # secrets that internal apps, such as the admin app or document download, must use to authenticate with the API ADMIN_CLIENT_ID = 'notify-admin' - GOVUK_ALERTS_CLIENT_ID = 'govuk-alerts' + GOVUK_ALERTS_CLIENT_ID = 'govuk-alerts' # TODO: can remove? INTERNAL_CLIENT_API_KEYS = json.loads( os.environ.get('INTERNAL_CLIENT_API_KEYS', '{"notify-admin":["dev-notify-secret-key"]}') - ) + ) # TODO: handled by varsfile? # encyption secret/salt - ADMIN_CLIENT_SECRET = os.getenv('ADMIN_CLIENT_SECRET', 'dev-notify-secret-key') - SECRET_KEY = os.getenv('SECRET_KEY', 'dev-notify-secret-key') - DANGEROUS_SALT = os.getenv('DANGEROUS_SALT', 'dev-notify-salt ') + ADMIN_CLIENT_SECRET = os.environ.get('ADMIN_CLIENT_SECRET') + SECRET_KEY = os.environ.get('SECRET_KEY') + DANGEROUS_SALT = os.environ.get('DANGEROUS_SALT') # DB conection string - SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_URI') + SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI') # AWS SMS - AWS_PINPOINT_REGION = os.getenv("AWS_PINPOINT_REGION", "us-west-2") - AWS_US_TOLL_FREE_NUMBER = os.getenv("AWS_US_TOLL_FREE_NUMBER", "+18446120782") + AWS_PINPOINT_REGION = os.environ.get("AWS_PINPOINT_REGION") + AWS_US_TOLL_FREE_NUMBER = os.environ.get("AWS_US_TOLL_FREE_NUMBER") # MMG API Key - MMG_API_KEY = os.getenv('MMG_API_KEY') + MMG_API_KEY = os.environ.get('MMG_API_KEY', 'placeholder') # Firetext API Key - FIRETEXT_API_KEY = os.getenv("FIRETEXT_API_KEY") - FIRETEXT_INTERNATIONAL_API_KEY = os.getenv("FIRETEXT_INTERNATIONAL_API_KEY", "placeholder") + FIRETEXT_API_KEY = os.environ.get("FIRETEXT_API_KEY", "placeholder") + FIRETEXT_INTERNATIONAL_API_KEY = os.environ.get("FIRETEXT_INTERNATIONAL_API_KEY", "placeholder") # Prefix to identify queues in SQS - NOTIFICATION_QUEUE_PREFIX = os.getenv('NOTIFICATION_QUEUE_PREFIX') + NOTIFICATION_QUEUE_PREFIX = os.environ.get('NOTIFICATION_QUEUE_PREFIX') # URL of redis instance - REDIS_URL = os.getenv('REDIS_URL') - REDIS_ENABLED = 1 + REDIS_URL = os.environ.get('REDIS_URL') + REDIS_ENABLED = True EXPIRE_CACHE_TEN_MINUTES = 600 EXPIRE_CACHE_EIGHT_DAYS = 8 * 24 * 60 * 60 @@ -128,7 +128,7 @@ class Config(object): # Logging DEBUG = False - NOTIFY_LOG_PATH = os.getenv('NOTIFY_LOG_PATH') + NOTIFY_LOG_PATH = os.environ.get('NOTIFY_LOG_PATH') # Cronitor CRONITOR_ENABLED = False @@ -349,12 +349,12 @@ class Config(object): } # we can set celeryd_prefetch_multiplier to be 1 for celery apps which handle only long running tasks - if os.getenv('CELERYD_PREFETCH_MULTIPLIER'): - CELERY['worker_prefetch_multiplier'] = os.getenv('CELERYD_PREFETCH_MULTIPLIER') + if os.environ.get('CELERYD_PREFETCH_MULTIPLIER'): + CELERY['worker_prefetch_multiplier'] = os.environ.get('CELERYD_PREFETCH_MULTIPLIER') FROM_NUMBER = 'development' - STATSD_HOST = os.getenv('STATSD_HOST') + STATSD_HOST = os.environ.get('STATSD_HOST') STATSD_PORT = 8125 STATSD_ENABLED = bool(STATSD_HOST) @@ -409,7 +409,7 @@ class Development(Config): DEBUG = True SQLALCHEMY_ECHO = False - REDIS_ENABLED = os.getenv('REDIS_ENABLED') == '1' + REDIS_ENABLED = True CSV_UPLOAD_BUCKET_NAME = 'local-notifications-csv-upload' CONTACT_LIST_BUCKET_NAME = 'local-contact-list' @@ -436,12 +436,12 @@ class Development(Config): NOTIFY_LOG_PATH = 'application.log' NOTIFY_EMAIL_DOMAIN = "dispostable.com" - SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_URI', 'postgresql://postgres:chummy@db:5432/notification_api') - REDIS_URL = os.getenv('REDIS_URL', 'redis://localhost:6379/0') + SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI', 'postgresql://postgres:chummy@db:5432/notification_api') + REDIS_URL = os.environ.get('REDIS_URL') - ANTIVIRUS_ENABLED = os.getenv('ANTIVIRUS_ENABLED') == '1' + ANTIVIRUS_ENABLED = os.environ.get('ANTIVIRUS_ENABLED') == '1' - API_HOST_NAME = os.getenv('API_HOST_NAME', 'http://localhost:6011') + API_HOST_NAME = os.environ.get('API_HOST_NAME', 'http://localhost:6011') API_RATE_LIMIT_ENABLED = True DVLA_EMAIL_ADDRESSES = ['success@simulator.amazonses.com'] @@ -472,7 +472,7 @@ class Test(Development): LETTER_SANITISE_BUCKET_NAME = 'test-letters-sanitise' # this is overriden in jenkins and on cloudfoundry - SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_URI', 'postgresql://postgres:chummy@db:5432/notification_api') + SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI', 'postgresql://postgres:chummy@db:5432/notification_api') CELERY = { **Config.CELERY, @@ -532,26 +532,29 @@ class Staging(Config): class Live(Config): NOTIFY_EMAIL_DOMAIN = os.environ.get('NOTIFY_EMAIL_DOMAIN') NOTIFY_ENVIRONMENT = 'live' - CSV_UPLOAD_BUCKET_NAME = 'live-notifications-csv-upload' - CONTACT_LIST_BUCKET_NAME = 'production-contact-list' - TEST_LETTERS_BUCKET_NAME = 'production-test-letters' - DVLA_RESPONSE_BUCKET_NAME = 'notifications.service.gov.uk-ftp' - LETTERS_PDF_BUCKET_NAME = 'production-letters-pdf' - LETTERS_SCAN_BUCKET_NAME = 'production-letters-scan' - INVALID_PDF_BUCKET_NAME = 'production-letters-invalid-pdf' - TRANSIENT_UPLOADED_LETTERS = 'production-transient-uploaded-letters' - LETTER_SANITISE_BUCKET_NAME = 'production-letters-sanitise' - FROM_NUMBER = 'GOVUK' + # buckets + CSV_UPLOAD_BUCKET_NAME = 'notifications-prototype-csv-upload' # created in gsa sandbox + CONTACT_LIST_BUCKET_NAME = 'notifications-prototype-contact-list-upload' # created in gsa sandbox + # TODO: verify below buckets only used for letters + TEST_LETTERS_BUCKET_NAME = 'production-test-letters' # not created in gsa sandbox + DVLA_RESPONSE_BUCKET_NAME = 'notifications.service.gov.uk-ftp' # not created in gsa sandbox + LETTERS_PDF_BUCKET_NAME = 'production-letters-pdf' # not created in gsa sandbox + LETTERS_SCAN_BUCKET_NAME = 'production-letters-scan' # not created in gsa sandbox + INVALID_PDF_BUCKET_NAME = 'production-letters-invalid-pdf' # not created in gsa sandbox + TRANSIENT_UPLOADED_LETTERS = 'production-transient-uploaded-letters' # not created in gsa sandbox + LETTER_SANITISE_BUCKET_NAME = 'production-letters-sanitise' # not created in gsa sandbox + + FROM_NUMBER = 'US Notify' API_RATE_LIMIT_ENABLED = True CHECK_PROXY_HEADER = True SES_STUB_URL = None CRONITOR_ENABLED = True # DEBUG = True - REDIS_ENABLED = os.getenv('REDIS_ENABLED') == '1' + REDIS_ENABLED = True - NOTIFY_LOG_PATH = os.getenv('NOTIFY_LOG_PATH', 'application.log') - REDIS_URL = os.getenv('REDIS_URL', 'redis://localhost:6379/0') + NOTIFY_LOG_PATH = os.environ.get('NOTIFY_LOG_PATH', 'application.log') + REDIS_URL = os.environ.get('REDIS_URL') class CloudFoundryConfig(Config): diff --git a/devcontainer-api/.devcontainer.json b/devcontainer-api/.devcontainer.json index 3126c6d12..265b505ff 100644 --- a/devcontainer-api/.devcontainer.json +++ b/devcontainer-api/.devcontainer.json @@ -23,9 +23,9 @@ } }, "extensions": [ - "ms-python.python", + // "ms-python.python", "donjayamanne.python-extension-pack", - "ms-azuretools.vscode-docker", + // "ms-azuretools.vscode-docker", "ms-python.vscode-pylance", "eamodio.gitlens", "wholroyd.jinja", @@ -33,10 +33,13 @@ "visualstudioexptteam.vscodeintellicode", "yzhang.markdown-all-in-one", "ms-ossdata.vscode-postgresql", - "GitHub.copilot", - "ms-vsliveshare.vsliveshare", - "mtxr.sqltools", - "mtxr.sqltools-driver-pg" + "GitHub.copilot" + // "ms-vsliveshare.vsliveshare", + // "mtxr.sqltools", + // "mtxr.sqltools-driver-pg" + ], + "forwardPorts": [ + 6011 ], "postCreateCommand": "notify-dev-entrypoint.sh", "remoteUser": "vscode" diff --git a/manifest.yml b/manifest.yml index 425440144..c1ca4f5c6 100644 --- a/manifest.yml +++ b/manifest.yml @@ -34,8 +34,6 @@ applications: SECRET_KEY: ((SECRET_KEY)) AWS_REGION: us-west-2 AWS_PINPOINT_REGION: us-west-2 - AWS_ACCESS_KEY_ID: ((AWS_ACCESS_KEY_ID)) - AWS_SECRET_ACCESS_KEY: ((AWS_SECRET_ACCESS_KEY)) AWS_US_TOLL_FREE_NUMBER: +18446120782 DVLA_EMAIL_ADDRESSES: [] diff --git a/varsfile.sample b/varsfile.sample index 5105b1880..9cd9a537a 100644 --- a/varsfile.sample +++ b/varsfile.sample @@ -1,16 +1,12 @@ SECRET_KEY: "dev-notify-secret-key" # pragma: allowlist secret DANGEROUS_SALT: "dev-notify-salt" -AWS_ACCESS_KEY_ID: -AWS_SECRET_ACCESS_KEY: ADMIN_BASE_URL: https://notifications-admin.app.cloud.gov ADMIN_CLIENT_ID: notify-admin ADMIN_CLIENT_SECRET: dev-notify-secret-key API_HOST_NAME: https://notifications-api.app.cloud.gov -AWS_ACCESS_KEY_ID: placeholder AWS_PINPOINT_REGION: us-west-2 AWS_REGION: us-west-2 -AWS_SECRET_ACCESS_KEY: placeholder AWS_US_TOLL_FREE_NUMBER: 18446120782 DANGEROUS_SALT: dev-notify-salt DVLA_EMAIL_ADDRESSES: []