From c1b00a5f0cb880008028dd72eee94c10c20c72ba Mon Sep 17 00:00:00 2001 From: Martyn Inglis Date: Wed, 7 Sep 2016 09:35:31 +0100 Subject: [PATCH 1/3] Simplified the config. Aim is to get the actual secrets in credstash to be env specific, and not the random collection of things we have at the moment. Secret definition also includes env specific things such as URLs / Queue prefixes / URLs for providers and so on. --- README.md | 36 +++++--------- app/__init__.py | 1 - config.py | 94 +++++++++++++++++++++++++---------- environment_test.sh | 29 ++++------- tests/app/clients/test_mmg.py | 2 +- 5 files changed, 91 insertions(+), 71 deletions(-) diff --git a/README.md b/README.md index a6917c43c..b8b992150 100644 --- a/README.md +++ b/README.md @@ -20,31 +20,19 @@ Create a local environment.sh file containing the following: ``` echo " -export NOTIFY_ENVIRONMENT='development' -export ADMIN_BASE_URL='http://localhost:6012' -export ADMIN_CLIENT_USER_NAME='dev-notify-admin' -export ADMIN_CLIENT_SECRET='dev-notify-secret-key' -export API_HOST_NAME='http://localhost:6011' - -export AWS_REGION='eu-west-1' -export AWS_ACCESS_KEY_ID=[MY ACCESS KEY] -export AWS_SECRET_ACCESS_KEY=[MY SECRET] - -export DANGEROUS_SALT='dev-notify-salt' -export FIRETEXT_API_KEY=[contact team member for api key] -export FROM_NUMBER='40605' -export INVITATION_EMAIL_FROM='invites' -export INVITATION_EXPIRATION_DAYS=2 -export MMG_API_KEY=mmg=secret-key -export MMG_URL="https://api.mmg.co.uk/json/api.php" -export NOTIFICATION_QUEUE_PREFIX='[unique-to-environment]' # -export NOTIFY_EMAIL_DOMAIN='notify.tools' -export SECRET_KEY='dev-notify-secret-key' export SQLALCHEMY_DATABASE_URI='postgresql://localhost/notification_api' -export STATSD_ENABLED=True -export STATSD_HOST="localhost" -export STATSD_PORT=1000 -export STATSD_PREFIX="stats-prefix" +export SECRET_KEY='secret-key' +export DANGEROUS_SALT='dangerous-salt' +export NOTIFY_ENVIRONMENT="development" +export ADMIN_CLIENT_SECRET='notify-secret-key' +export ADMIN_BASE_URL='http://localhost:6012' +export FROM_NUMBER='development' +export MMG_URL="https://api.mmg.co.uk/json/api.php" +export MMG_API_KEY='MMG_API_KEY' +export LOADTESTING_API_KEY="FIRETEXT_SIMULATION_KEY" +export FIRETEXT_API_KEY="FIRETEXT_ACTUAL_KEY" +export STATSD_PREFIX="FAKE_PREFIX" +export NOTIFICATION_QUEUE_PREFIX="PREFIX-TO-IDENTIFY-SQS-QUEUE" "> environment.sh ``` diff --git a/app/__init__.py b/app/__init__.py index 464157f60..cf47632f9 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -41,7 +41,6 @@ def create_app(app_name=None): from config import configs application.config.from_object(configs[os.environ['NOTIFY_ENVIRONMENT']]) - if app_name: application.config['NOTIFY_APP_NAME'] = app_name diff --git a/config.py b/config.py index 44a0b88d1..32395c5d7 100644 --- a/config.py +++ b/config.py @@ -5,27 +5,61 @@ import os class Config(object): - DEBUG = False + ######################################## + # Secrets that are held in credstash ### + ######################################## + + # URL of admin app ADMIN_BASE_URL = os.environ['ADMIN_BASE_URL'] - ADMIN_CLIENT_USER_NAME = os.environ['ADMIN_CLIENT_USER_NAME'] + + # admin app api key ADMIN_CLIENT_SECRET = os.environ['ADMIN_CLIENT_SECRET'] - AWS_REGION = os.environ['AWS_REGION'] + + # encyption secret/salt + SECRET_KEY = os.environ['SECRET_KEY'] DANGEROUS_SALT = os.environ['DANGEROUS_SALT'] - INVITATION_EXPIRATION_DAYS = int(os.environ['INVITATION_EXPIRATION_DAYS']) - INVITATION_EMAIL_FROM = os.environ['INVITATION_EMAIL_FROM'] + + # DB conection string + SQLALCHEMY_DATABASE_URI = os.environ['SQLALCHEMY_DATABASE_URI'] + + # MMG API Url + MMG_URL = os.environ['MMG_URL'] + + # MMG API Key + MMG_API_KEY = os.environ['MMG_API_KEY'] + + # Firetext API Key + FIRETEXT_API_KEY = os.getenv("FIRETEXT_API_KEY") + + # Firetext simluation key + LOADTESTING_API_KEY = os.getenv("LOADTESTING_API_KEY") + + # Hosted graphite statsd prefix + STATSD_PREFIX = os.getenv('STATSD_PREFIX') + + # Prefix to identify queues in SQS + NOTIFICATION_QUEUE_PREFIX = os.getenv('NOTIFICATION_QUEUE_PREFIX') + + ########################### + # Default config values ### + ########################### + + DEBUG = False + NOTIFY_ENVIRONMENT = 'development' + ADMIN_CLIENT_USER_NAME = 'notify-admin' + AWS_REGION = 'eu-west-1' + INVITATION_EXPIRATION_DAYS = 2 + INVITATION_EMAIL_FROM = 'no-reply' NOTIFY_APP_NAME = 'api' NOTIFY_LOG_PATH = '/var/log/notify/application.log' # Notification Queue names are a combination of a prefix plus a name - NOTIFICATION_QUEUE_PREFIX = os.environ['NOTIFICATION_QUEUE_PREFIX'] - SECRET_KEY = os.environ['SECRET_KEY'] + NOTIFICATION_QUEUE_PREFIX = 'development' SQLALCHEMY_COMMIT_ON_TEARDOWN = False - SQLALCHEMY_DATABASE_URI = os.environ['SQLALCHEMY_DATABASE_URI'] SQLALCHEMY_RECORD_QUERIES = True SQLALCHEMY_TRACK_MODIFICATIONS = True - NOTIFY_EMAIL_DOMAIN = os.environ['NOTIFY_EMAIL_DOMAIN'] + NOTIFY_EMAIL_DOMAIN = 'notify.tools' PAGE_SIZE = 50 SMS_CHAR_COUNT_LIMIT = 495 - MMG_URL = os.environ['MMG_URL'] BRANDING_PATH = '/static/images/email-template/crests/' NOTIFY_SERVICE_ID = 'd6aa2c68-a2d9-4437-ab19-3ae8eb202553' @@ -37,10 +71,10 @@ class Config(object): BROKER_URL = 'sqs://' BROKER_TRANSPORT_OPTIONS = { - 'region': 'eu-west-1', + 'region': AWS_REGION, 'polling_interval': 1, # 1 second 'visibility_timeout': 14410, # 4 hours 10 seconds. 10 seconds longer than max retry - 'queue_name_prefix': os.environ['NOTIFICATION_QUEUE_PREFIX'] + '-' + 'queue_name_prefix': NOTIFICATION_QUEUE_PREFIX } CELERY_ENABLE_UTC = True, CELERY_TIMEZONE = 'Europe/London' @@ -96,26 +130,26 @@ class Config(object): Queue('retry', Exchange('default'), routing_key='retry'), Queue('email-already-registered', Exchange('default'), routing_key='email-already-registered') ] - API_HOST_NAME = os.environ['API_HOST_NAME'] - MMG_API_KEY = os.environ['MMG_API_KEY'] - FIRETEXT_API_KEY = os.getenv("FIRETEXT_API_KEY") - LOADTESTING_NUMBER = os.getenv('LOADTESTING_NUMBER') - LOADTESTING_API_KEY = os.getenv("LOADTESTING_API_KEY") - CSV_UPLOAD_BUCKET_NAME = os.getenv("CSV_UPLOAD_BUCKET_NAME") + API_HOST_NAME = "http://localhost:6011" + + CSV_UPLOAD_BUCKET_NAME = 'development-notifications-csv-upload' NOTIFICATIONS_ALERT = 5 # five mins - FROM_NUMBER = os.getenv('FROM_NUMBER') + FROM_NUMBER = 'development' STATSD_ENABLED = False STATSD_HOST = "statsd.hostedgraphite.com" STATSD_PORT = 8125 - STATSD_PREFIX = None SENDING_NOTIFICATIONS_TIMEOUT_PERIOD = 259200 +###################### +# Config overrides ### +###################### + class Development(Config): NOTIFY_ENVIRONMENT = 'development' - CSV_UPLOAD_BUCKET_NAME = 'development-notifications-csv-upload' + NOTIFICATION_QUEUE_PREFIX = 'development' DEBUG = True SQLALCHEMY_ECHO = False CELERY_QUEUES = Config.CELERY_QUEUES + [ @@ -127,10 +161,14 @@ class Development(Config): class Test(Config): + NOTIFY_EMAIL_DOMAIN = 'test.notify.com' + FROM_NUMBER = 'testing' NOTIFY_ENVIRONMENT = 'test' DEBUG = True CSV_UPLOAD_BUCKET_NAME = 'test-notifications-csv-upload' - STATSD_PREFIX = "test" + STATSD_ENABLED = True + STATSD_HOST = "localhost" + STATSD_PORT = 1000 CELERY_QUEUES = Config.CELERY_QUEUES + [ Queue('db-sms', Exchange('default'), routing_key='db-sms'), Queue('send-sms', Exchange('default'), routing_key='send-sms'), @@ -140,23 +178,29 @@ class Test(Config): class Preview(Config): + NOTIFY_EMAIL_DOMAIN = 'notify.works' NOTIFY_ENVIRONMENT = 'preview' CSV_UPLOAD_BUCKET_NAME = 'preview-notifications-csv-upload' - STATSD_PREFIX = "preview" + API_HOST_NAME = 'http://admin-api.internal' + FROM_NUMBER = 'NotifyPreview' class Staging(Config): + NOTIFY_EMAIL_DOMAIN = 'staging-notify.works' NOTIFY_ENVIRONMENT = 'staging' CSV_UPLOAD_BUCKET_NAME = 'staging-notify-csv-upload' - STATSD_PREFIX = os.getenv('STATSD_PREFIX') STATSD_ENABLED = True + API_HOST_NAME = 'http://admin-api.internal' + FROM_NUMBER = 'NotifyStage' class Live(Config): + NOTIFY_EMAIL_DOMAIN = 'notifications.service.gov.uk' NOTIFY_ENVIRONMENT = 'live' CSV_UPLOAD_BUCKET_NAME = 'live-notifications-csv-upload' - STATSD_PREFIX = os.getenv('STATSD_PREFIX') STATSD_ENABLED = True + API_HOST_NAME = 'http://admin-api.internal' + FROM_NUMBER = '40604' configs = { diff --git a/environment_test.sh b/environment_test.sh index d0c536072..3f8a2fbe5 100644 --- a/environment_test.sh +++ b/environment_test.sh @@ -1,24 +1,13 @@ #!/bin/bash -export NOTIFY_ENVIRONMENT='test' -export ADMIN_BASE_URL='http://localhost:6012' -export ADMIN_CLIENT_USER_NAME='dev-notify-admin' -export ADMIN_CLIENT_SECRET='dev-notify-secret-key' -export AWS_REGION='eu-west-1' -export DANGEROUS_SALT='dangerous-salt' -export INVITATION_EMAIL_FROM='invites' -export INVITATION_EXPIRATION_DAYS=2 -export NOTIFICATION_QUEUE_PREFIX='test-env-not-used' -export SECRET_KEY='secret-key' export SQLALCHEMY_DATABASE_URI=${TEST_DATABASE:='postgresql://localhost/test_notification_api'} -export FIRETEXT_API_KEY="Firetext" -export NOTIFY_EMAIL_DOMAIN="test.notify.com" -export MMG_API_KEY='mmg-secret-key' -export LOADTESTING_API_KEY="loadtesting" -export LOADTESTING_NUMBER="loadtesting" -export STATSD_ENABLED=True -export STATSD_HOST="localhost" -export STATSD_PORT=1000 -export STATSD_PREFIX="stats-prefix" -export API_HOST_NAME="http://localhost:6011" +export SECRET_KEY='secret-key' +export DANGEROUS_SALT='dangerous-salt' +export NOTIFY_ENVIRONMENT='test' +export ADMIN_CLIENT_SECRET='dev-notify-secret-key' +export ADMIN_BASE_URL='http://localhost:6012' export FROM_NUMBER='from_number' export MMG_URL="https://api.mmg.co.uk/json/api.php" +export MMG_API_KEY='mmg-secret-key' +export LOADTESTING_API_KEY="loadtesting" +export FIRETEXT_API_KEY="Firetext" +export STATSD_PREFIX="stats-prefix" diff --git a/tests/app/clients/test_mmg.py b/tests/app/clients/test_mmg.py index bc4428193..649764d89 100644 --- a/tests/app/clients/test_mmg.py +++ b/tests/app/clients/test_mmg.py @@ -60,7 +60,7 @@ def test_send_sms_calls_mmg_correctly(notify_api, mocker): assert request_args['reqType'] == 'BULK' assert request_args['MSISDN'] == to assert request_args['msg'] == content - assert request_args['sender'] == 'from_number' + assert request_args['sender'] == 'testing' assert request_args['cid'] == reference assert request_args['multi'] is True From d8bfad1ce7e5f116af5f11010f60e84750ddec5c Mon Sep 17 00:00:00 2001 From: Martyn Inglis Date: Wed, 7 Sep 2016 09:37:45 +0100 Subject: [PATCH 2/3] Added prefix to test env file for completeness --- environment_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment_test.sh b/environment_test.sh index 3f8a2fbe5..a74a81a86 100644 --- a/environment_test.sh +++ b/environment_test.sh @@ -1,4 +1,3 @@ -#!/bin/bash export SQLALCHEMY_DATABASE_URI=${TEST_DATABASE:='postgresql://localhost/test_notification_api'} export SECRET_KEY='secret-key' export DANGEROUS_SALT='dangerous-salt' @@ -11,3 +10,4 @@ export MMG_API_KEY='mmg-secret-key' export LOADTESTING_API_KEY="loadtesting" export FIRETEXT_API_KEY="Firetext" export STATSD_PREFIX="stats-prefix" +export NOTIFICATION_QUEUE_PREFIX='testing' From bdc1d464bf59c5259f727de61db149e158a029c0 Mon Sep 17 00:00:00 2001 From: Martyn Inglis Date: Thu, 8 Sep 2016 09:07:43 +0100 Subject: [PATCH 3/3] Fixed up a few things as per pull request review: - Moved email domain and css bucket into dev config, not default - deleted duplicated property - removed unused property --- config.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/config.py b/config.py index 32395c5d7..d4d43363a 100644 --- a/config.py +++ b/config.py @@ -49,15 +49,11 @@ class Config(object): ADMIN_CLIENT_USER_NAME = 'notify-admin' AWS_REGION = 'eu-west-1' INVITATION_EXPIRATION_DAYS = 2 - INVITATION_EMAIL_FROM = 'no-reply' NOTIFY_APP_NAME = 'api' NOTIFY_LOG_PATH = '/var/log/notify/application.log' - # Notification Queue names are a combination of a prefix plus a name - NOTIFICATION_QUEUE_PREFIX = 'development' SQLALCHEMY_COMMIT_ON_TEARDOWN = False SQLALCHEMY_RECORD_QUERIES = True SQLALCHEMY_TRACK_MODIFICATIONS = True - NOTIFY_EMAIL_DOMAIN = 'notify.tools' PAGE_SIZE = 50 SMS_CHAR_COUNT_LIMIT = 495 BRANDING_PATH = '/static/images/email-template/crests/' @@ -132,7 +128,6 @@ class Config(object): ] API_HOST_NAME = "http://localhost:6011" - CSV_UPLOAD_BUCKET_NAME = 'development-notifications-csv-upload' NOTIFICATIONS_ALERT = 5 # five mins FROM_NUMBER = 'development' @@ -148,6 +143,8 @@ class Config(object): ###################### class Development(Config): + NOTIFY_EMAIL_DOMAIN = 'notify.tools' + CSV_UPLOAD_BUCKET_NAME = 'development-notifications-csv-upload' NOTIFY_ENVIRONMENT = 'development' NOTIFICATION_QUEUE_PREFIX = 'development' DEBUG = True