From 18a04fd7199f0eb60662e8bf4f754baf3ec96901 Mon Sep 17 00:00:00 2001 From: James Moffet Date: Mon, 25 Jul 2022 15:19:05 -0700 Subject: [PATCH] configs --- Makefile | 10 ++-- README.md | 35 ++++++------- app/config.py | 81 +++++++++++++++-------------- devcontainer-api/.devcontainer.json | 15 +++--- manifest.yml | 2 - varsfile.sample | 4 -- 6 files changed, 71 insertions(+), 76 deletions(-) diff --git a/Makefile b/Makefile index c1aaeb999..0faec389f 100644 --- a/Makefile +++ b/Makefile @@ -146,16 +146,16 @@ cf-deploy: ## Deploys the app to Cloud Foundry cf-deploy-api-db-migration: $(if ${CF_SPACE},,$(error Must specify CF_SPACE)) cf target -o ${CF_ORG} -s ${CF_SPACE} - make -s CF_APP=notify-api-db-migration generate-manifest > ${CF_MANIFEST_PATH} + make -s CF_APP=notifications-api generate-manifest > ${CF_MANIFEST_PATH} - cf push notify-api-db-migration --no-route -f ${CF_MANIFEST_PATH} + cf push notifications-api --no-route -f ${CF_MANIFEST_PATH} rm ${CF_MANIFEST_PATH} - cf run-task notify-api-db-migration --command="flask db upgrade" --name api_db_migration + cf run-task notifications-api --command="flask db upgrade" --name api_db_migration .PHONY: cf-check-api-db-migration-task -cf-check-api-db-migration-task: ## Get the status for the last notify-api-db-migration task - @cf curl /v3/apps/`cf app --guid notify-api-db-migration`/tasks?order_by=-created_at | jq -r ".resources[0].state" +cf-check-api-db-migration-task: ## Get the status for the last notifications-api task + @cf curl /v3/apps/`cf app --guid notifications-api`/tasks?order_by=-created_at | jq -r ".resources[0].state" .PHONY: cf-rollback cf-rollback: ## Rollbacks the app to the previous release diff --git a/README.md b/README.md index 7f50a00bd..6ce7e77ac 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ The command that is actually run by the pre-commit hook is: `git diff --staged - You can also run against all tracked files staged or not: `git ls-files -z | xargs -0 detect-secrets-hook --baseline .secrets.baseline` -### Postgres +### Postgres [DEPRECATED] Install [Postgres.app](http://postgresapp.com/). @@ -84,7 +84,7 @@ Currently the API works with PostgreSQL 11. After installation, open the Postgre export PATH=${PATH}:/Applications/Postgres.app/Contents/Versions/11/bin/ ``` -### Redis +### Redis [DEPRECATED] To switch redis on you'll need to install it locally. On a Mac you can do: @@ -103,32 +103,27 @@ export REDIS_ENABLED=1 ## To run the application ``` -# install dependencies, etc. -make bootstrap +# set up AWS SES/SNS as instructed above -# run the web app +# create .env file as instructed above + +# download vscode and install the Remote-Containers plug-in from Microsoft + +# create the external docker network +docker network create notify-network + +# Using the command pallette, search "Remote Containers: Open folder in project" and choose devcontainer-api, then wait for docker to build + +# Open a terminal in vscode and run the web server make run-flask -# run the background tasks +# Open another terminal in vscode and run the background tasks make run-celery -# run scheduled tasks (optional) +# Open a third terminal in vscode and run scheduled tasks (optional) make run-celery-beat ``` -We've had problems running Celery locally due to one of its dependencies: pycurl. Due to the complexity of the issue, we also support running Celery via Docker: - -``` -# install dependencies, etc. -make bootstrap-with-docker - -# run the background tasks -make run-celery-with-docker - -# run scheduled tasks -make run-celery-beat-with-docker -``` - ## To test the application ``` diff --git a/app/config.py b/app/config.py index f5f7f9325..5b2fb1dca 100644 --- a/app/config.py +++ b/app/config.py @@ -82,44 +82,44 @@ class TaskNames(object): class Config(object): # URL of admin app - ADMIN_BASE_URL = os.getenv('ADMIN_BASE_URL') + ADMIN_BASE_URL = os.environ.get('ADMIN_BASE_URL') # URL of api app (on AWS this is the internal api endpoint) - API_HOST_NAME = os.getenv('API_HOST_NAME') + API_HOST_NAME = os.environ.get('API_HOST_NAME') # secrets that internal apps, such as the admin app or document download, must use to authenticate with the API ADMIN_CLIENT_ID = 'notify-admin' - GOVUK_ALERTS_CLIENT_ID = 'govuk-alerts' + GOVUK_ALERTS_CLIENT_ID = 'govuk-alerts' # TODO: can remove? INTERNAL_CLIENT_API_KEYS = json.loads( os.environ.get('INTERNAL_CLIENT_API_KEYS', '{"notify-admin":["dev-notify-secret-key"]}') - ) + ) # TODO: handled by varsfile? # encyption secret/salt - ADMIN_CLIENT_SECRET = os.getenv('ADMIN_CLIENT_SECRET', 'dev-notify-secret-key') - SECRET_KEY = os.getenv('SECRET_KEY', 'dev-notify-secret-key') - DANGEROUS_SALT = os.getenv('DANGEROUS_SALT', 'dev-notify-salt ') + ADMIN_CLIENT_SECRET = os.environ.get('ADMIN_CLIENT_SECRET') + SECRET_KEY = os.environ.get('SECRET_KEY') + DANGEROUS_SALT = os.environ.get('DANGEROUS_SALT') # DB conection string - SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_URI') + SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI') # AWS SMS - AWS_PINPOINT_REGION = os.getenv("AWS_PINPOINT_REGION", "us-west-2") - AWS_US_TOLL_FREE_NUMBER = os.getenv("AWS_US_TOLL_FREE_NUMBER", "+18446120782") + AWS_PINPOINT_REGION = os.environ.get("AWS_PINPOINT_REGION") + AWS_US_TOLL_FREE_NUMBER = os.environ.get("AWS_US_TOLL_FREE_NUMBER") # MMG API Key - MMG_API_KEY = os.getenv('MMG_API_KEY') + MMG_API_KEY = os.environ.get('MMG_API_KEY', 'placeholder') # Firetext API Key - FIRETEXT_API_KEY = os.getenv("FIRETEXT_API_KEY") - FIRETEXT_INTERNATIONAL_API_KEY = os.getenv("FIRETEXT_INTERNATIONAL_API_KEY", "placeholder") + FIRETEXT_API_KEY = os.environ.get("FIRETEXT_API_KEY", "placeholder") + FIRETEXT_INTERNATIONAL_API_KEY = os.environ.get("FIRETEXT_INTERNATIONAL_API_KEY", "placeholder") # Prefix to identify queues in SQS - NOTIFICATION_QUEUE_PREFIX = os.getenv('NOTIFICATION_QUEUE_PREFIX') + NOTIFICATION_QUEUE_PREFIX = os.environ.get('NOTIFICATION_QUEUE_PREFIX') # URL of redis instance - REDIS_URL = os.getenv('REDIS_URL') - REDIS_ENABLED = 1 + REDIS_URL = os.environ.get('REDIS_URL') + REDIS_ENABLED = True EXPIRE_CACHE_TEN_MINUTES = 600 EXPIRE_CACHE_EIGHT_DAYS = 8 * 24 * 60 * 60 @@ -128,7 +128,7 @@ class Config(object): # Logging DEBUG = False - NOTIFY_LOG_PATH = os.getenv('NOTIFY_LOG_PATH') + NOTIFY_LOG_PATH = os.environ.get('NOTIFY_LOG_PATH') # Cronitor CRONITOR_ENABLED = False @@ -349,12 +349,12 @@ class Config(object): } # we can set celeryd_prefetch_multiplier to be 1 for celery apps which handle only long running tasks - if os.getenv('CELERYD_PREFETCH_MULTIPLIER'): - CELERY['worker_prefetch_multiplier'] = os.getenv('CELERYD_PREFETCH_MULTIPLIER') + if os.environ.get('CELERYD_PREFETCH_MULTIPLIER'): + CELERY['worker_prefetch_multiplier'] = os.environ.get('CELERYD_PREFETCH_MULTIPLIER') FROM_NUMBER = 'development' - STATSD_HOST = os.getenv('STATSD_HOST') + STATSD_HOST = os.environ.get('STATSD_HOST') STATSD_PORT = 8125 STATSD_ENABLED = bool(STATSD_HOST) @@ -409,7 +409,7 @@ class Development(Config): DEBUG = True SQLALCHEMY_ECHO = False - REDIS_ENABLED = os.getenv('REDIS_ENABLED') == '1' + REDIS_ENABLED = True CSV_UPLOAD_BUCKET_NAME = 'local-notifications-csv-upload' CONTACT_LIST_BUCKET_NAME = 'local-contact-list' @@ -436,12 +436,12 @@ class Development(Config): NOTIFY_LOG_PATH = 'application.log' NOTIFY_EMAIL_DOMAIN = "dispostable.com" - SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_URI', 'postgresql://postgres:chummy@db:5432/notification_api') - REDIS_URL = os.getenv('REDIS_URL', 'redis://localhost:6379/0') + SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI', 'postgresql://postgres:chummy@db:5432/notification_api') + REDIS_URL = os.environ.get('REDIS_URL') - ANTIVIRUS_ENABLED = os.getenv('ANTIVIRUS_ENABLED') == '1' + ANTIVIRUS_ENABLED = os.environ.get('ANTIVIRUS_ENABLED') == '1' - API_HOST_NAME = os.getenv('API_HOST_NAME', 'http://localhost:6011') + API_HOST_NAME = os.environ.get('API_HOST_NAME', 'http://localhost:6011') API_RATE_LIMIT_ENABLED = True DVLA_EMAIL_ADDRESSES = ['success@simulator.amazonses.com'] @@ -472,7 +472,7 @@ class Test(Development): LETTER_SANITISE_BUCKET_NAME = 'test-letters-sanitise' # this is overriden in jenkins and on cloudfoundry - SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_URI', 'postgresql://postgres:chummy@db:5432/notification_api') + SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI', 'postgresql://postgres:chummy@db:5432/notification_api') CELERY = { **Config.CELERY, @@ -532,26 +532,29 @@ class Staging(Config): class Live(Config): NOTIFY_EMAIL_DOMAIN = os.environ.get('NOTIFY_EMAIL_DOMAIN') NOTIFY_ENVIRONMENT = 'live' - CSV_UPLOAD_BUCKET_NAME = 'live-notifications-csv-upload' - CONTACT_LIST_BUCKET_NAME = 'production-contact-list' - TEST_LETTERS_BUCKET_NAME = 'production-test-letters' - DVLA_RESPONSE_BUCKET_NAME = 'notifications.service.gov.uk-ftp' - LETTERS_PDF_BUCKET_NAME = 'production-letters-pdf' - LETTERS_SCAN_BUCKET_NAME = 'production-letters-scan' - INVALID_PDF_BUCKET_NAME = 'production-letters-invalid-pdf' - TRANSIENT_UPLOADED_LETTERS = 'production-transient-uploaded-letters' - LETTER_SANITISE_BUCKET_NAME = 'production-letters-sanitise' - FROM_NUMBER = 'GOVUK' + # buckets + CSV_UPLOAD_BUCKET_NAME = 'notifications-prototype-csv-upload' # created in gsa sandbox + CONTACT_LIST_BUCKET_NAME = 'notifications-prototype-contact-list-upload' # created in gsa sandbox + # TODO: verify below buckets only used for letters + TEST_LETTERS_BUCKET_NAME = 'production-test-letters' # not created in gsa sandbox + DVLA_RESPONSE_BUCKET_NAME = 'notifications.service.gov.uk-ftp' # not created in gsa sandbox + LETTERS_PDF_BUCKET_NAME = 'production-letters-pdf' # not created in gsa sandbox + LETTERS_SCAN_BUCKET_NAME = 'production-letters-scan' # not created in gsa sandbox + INVALID_PDF_BUCKET_NAME = 'production-letters-invalid-pdf' # not created in gsa sandbox + TRANSIENT_UPLOADED_LETTERS = 'production-transient-uploaded-letters' # not created in gsa sandbox + LETTER_SANITISE_BUCKET_NAME = 'production-letters-sanitise' # not created in gsa sandbox + + FROM_NUMBER = 'US Notify' API_RATE_LIMIT_ENABLED = True CHECK_PROXY_HEADER = True SES_STUB_URL = None CRONITOR_ENABLED = True # DEBUG = True - REDIS_ENABLED = os.getenv('REDIS_ENABLED') == '1' + REDIS_ENABLED = True - NOTIFY_LOG_PATH = os.getenv('NOTIFY_LOG_PATH', 'application.log') - REDIS_URL = os.getenv('REDIS_URL', 'redis://localhost:6379/0') + NOTIFY_LOG_PATH = os.environ.get('NOTIFY_LOG_PATH', 'application.log') + REDIS_URL = os.environ.get('REDIS_URL') class CloudFoundryConfig(Config): diff --git a/devcontainer-api/.devcontainer.json b/devcontainer-api/.devcontainer.json index 3126c6d12..265b505ff 100644 --- a/devcontainer-api/.devcontainer.json +++ b/devcontainer-api/.devcontainer.json @@ -23,9 +23,9 @@ } }, "extensions": [ - "ms-python.python", + // "ms-python.python", "donjayamanne.python-extension-pack", - "ms-azuretools.vscode-docker", + // "ms-azuretools.vscode-docker", "ms-python.vscode-pylance", "eamodio.gitlens", "wholroyd.jinja", @@ -33,10 +33,13 @@ "visualstudioexptteam.vscodeintellicode", "yzhang.markdown-all-in-one", "ms-ossdata.vscode-postgresql", - "GitHub.copilot", - "ms-vsliveshare.vsliveshare", - "mtxr.sqltools", - "mtxr.sqltools-driver-pg" + "GitHub.copilot" + // "ms-vsliveshare.vsliveshare", + // "mtxr.sqltools", + // "mtxr.sqltools-driver-pg" + ], + "forwardPorts": [ + 6011 ], "postCreateCommand": "notify-dev-entrypoint.sh", "remoteUser": "vscode" diff --git a/manifest.yml b/manifest.yml index 425440144..c1ca4f5c6 100644 --- a/manifest.yml +++ b/manifest.yml @@ -34,8 +34,6 @@ applications: SECRET_KEY: ((SECRET_KEY)) AWS_REGION: us-west-2 AWS_PINPOINT_REGION: us-west-2 - AWS_ACCESS_KEY_ID: ((AWS_ACCESS_KEY_ID)) - AWS_SECRET_ACCESS_KEY: ((AWS_SECRET_ACCESS_KEY)) AWS_US_TOLL_FREE_NUMBER: +18446120782 DVLA_EMAIL_ADDRESSES: [] diff --git a/varsfile.sample b/varsfile.sample index 5105b1880..9cd9a537a 100644 --- a/varsfile.sample +++ b/varsfile.sample @@ -1,16 +1,12 @@ SECRET_KEY: "dev-notify-secret-key" # pragma: allowlist secret DANGEROUS_SALT: "dev-notify-salt" -AWS_ACCESS_KEY_ID: -AWS_SECRET_ACCESS_KEY: ADMIN_BASE_URL: https://notifications-admin.app.cloud.gov ADMIN_CLIENT_ID: notify-admin ADMIN_CLIENT_SECRET: dev-notify-secret-key API_HOST_NAME: https://notifications-api.app.cloud.gov -AWS_ACCESS_KEY_ID: placeholder AWS_PINPOINT_REGION: us-west-2 AWS_REGION: us-west-2 -AWS_SECRET_ACCESS_KEY: placeholder AWS_US_TOLL_FREE_NUMBER: 18446120782 DANGEROUS_SALT: dev-notify-salt DVLA_EMAIL_ADDRESSES: []