Merge pull request #573 from alphagov/new-staging

New staging
This commit is contained in:
minglis
2016-08-02 14:53:58 +01:00
committed by GitHub
14 changed files with 65 additions and 257 deletions

View File

@@ -24,143 +24,3 @@ notifications:
slack:
rooms:
secure: vCmpAjdXkprqGzSnQzcco1lNiFuj/nRQdlVTGSnvg2HEqZUM5vzmWtQYw3WNS/bcgbVxV/nLgza17cdsmFqOuaZll4kDM0swnmk17eynImAHPFF/flGsNcw4oRV7WzWrq7j1TvGzJzxJkHYOjJFk0iMLM7+P76sWHI6Oa5NHQYH2l7a4hqzqmuUKaloHeZX/Ro2RmFE+/t/SfNr0pmtTRO4CfLfPUdUs9D8Mx0cyc1Z9Ke4Cq+R2Lsp8BwH0nye6pb9cI4h1YmUY/Xt8O1Z0QbncHZviWojSB2vV0V5WQPzqjjseAznJ1depK/LPOlRHlqddkEbGK28rDwsxbzY1q3DCZZiZZTXBD80HyNr/J4rfRAXhoCafwR57cqqIQ7G/fb5A/ckUM8TFKqUPh7wvu67OAiSWJmO78C8fumF3HxcoGp7DQOQPFLtWGuCfh2zjI9WbUhiHQwWl9/9qpgMEuXH7LOJAzgQbEp1A7XRmT2GupYEkGdFMwT8ojp8OXds08diEy1FNonWAtFZs8pO92urK862pCRB8WRQZWMjpcnif1Ht4QtwYFO9iVp5ygReTEoFuiQPMAypCVCzKgHe/W8LX55xzSnlUQqDU2NzN37LfC4ctS1h3BfIIU5KP9PmdgkKRUWcrFE3eMXPG36OdU4hRj4FMURzmAOZYhaSXuP8=
deploy:
- provider: s3
access_key_id: AKIAJQPPNM6P6V53SWKA
secret_access_key: &1
secure: irusc9JdTbXhKPMYUEHIOPXk7cCCleRsXXW/ZOAEWcHtdtuF6YNACsN3pywwyZ9UU+MqZb8lD3UH77o3u7BtuyPPv5JDy7cB7AoIX6EAaZIdxAS4v2tTpLPh6oS2QtfI1U5giI2g2C5IkRZIIrqiWtT63y0eReL5z5r5x7kEY5U7UOMrIL0w2m/frate0a4Q380ZbjRA1X0PcQ3IeavxQbTEYug+XEaKcpjqIFFwxeNYu814Ox7fMm+bqpG7TuwDF2i1+jAlX3NZSiuUBBYpHqgK6EARnCNwi8Yr2+CBO6ykXY1lcVn13+cSsFXDP7lgkK/BJUewWIrkQ6EL28ntTaG3cpZhSMB2fc21VRNnVzNDfzf9R+bI/Xy45fyraTVZ2Xun8j262SAM+F8D6Ivt1Y+n8aRHN0U8P//Yhfx2TMTFoHLhGUp076FDPKmcWRWpYikMzmTR/oVu6yE5OacntYBp8dE1QGrVSR//rRHK6QkQeDbnp/cSbOiZY92PeYeDjCLbZMrqASONWtzR6V0CPu/++iLw0mODd2xRlpsnFcpF8ygxFtAgOUrKXbAdu9JNdNE1pPE2FMDb6DKAuIxrRQT8ceUwhEChR7UGERTtbr2deyc+4f+MpmUK59uaqdG6y5q0KsiXhUTyzmKVLGxr/dyA5cATtsFDCx8caM0Qx6I=
local_dir: dpl_cd_upload
skip_cleanup: true
region: eu-west-1
bucket: notifications-api-codedeploy
wait-until-deployed: true
on:
repo: alphagov/notifications-api
- provider: codedeploy
access_key_id: AKIAJQPPNM6P6V53SWKA
secret_access_key: *1
bucket: notifications-api-codedeploy
key: notifications-api-$TRAVIS_BRANCH-$TRAVIS_BUILD_NUMBER-$TRAVIS_COMMIT.zip
bundle_type: zip
application: notifications-api
deployment_group: notifications_api_deployment_group
region: eu-west-1
wait-until-deployed: true
on: *2
- provider: codedeploy
access_key_id: AKIAJQPPNM6P6V53SWKA
secret_access_key: *1
bucket: notifications-api-codedeploy
key: notifications-api-$TRAVIS_BRANCH-$TRAVIS_BUILD_NUMBER-$TRAVIS_COMMIT.zip
bundle_type: zip
application: notifications-api
deployment_group: notifications_admin_api_deployment_group
region: eu-west-1
wait-until-deployed: true
on: *2
- provider: codedeploy
access_key_id: AKIAJQPPNM6P6V53SWKA
secret_access_key: *1
bucket: notifications-api-codedeploy
key: notifications-api-$TRAVIS_BRANCH-$TRAVIS_BUILD_NUMBER-$TRAVIS_COMMIT.zip
bundle_type: zip
application: notifications-api
deployment_group: notifications_delivery_api_deployment_group
region: eu-west-1
wait-until-deployed: true
on: *2
- provider: s3
access_key_id: AKIAJ5MKF6G3P2JQP4QQ
secret_access_key: &1
secure: daC1bCHXqLRK+iIZ8P699KCnTh77lwV4KxrZxL1yd6cstgfptyd/rg1WgRwE6QdxOCT9gQvKWUZFCzFy7M6E/Ih8EUHqEXTzC5M4oAye8rhePIBMQwqkgfYyIoZ3LdDMMP5JfBhiz0zS3Vj7HerL2qIu12adJBjkRJx3XAGimCrFOMQ0xUXQAKDjL6Xmv+gVz2f/ISLy6icKY4KNGt3cQV+8pa5aMF34C9R2udA9N67EWlXlh7hJbFtmY+0Zqpo8Rr6wKRb5MA0xEcTVLORSz1aa6GkxUCbzaIH99p7z3Ghz0qW2bUi9ZcDrvg0GLbVe1T+1HXhfktJfW8wnzw6A/2U/CIIFDQZ/qk0w/DkEwpQinXow99Zl49CcEU+v8llKhg5nM3LmAZCQg1c/iZyP/d90AwAMoMA/VTDD72M93IqTJQH18eC8g02DwE0hNDD6aos5wzeuDeiH/6BG+Tq0pDl0y0aWCcHf3vGRlo/5GlWfpE0vMQEC+qnEOWOUqSprCdSypgD2Aip9mCC98w4BkqKKvGNHPZolA7rxf7E9hTK+BNPRATpYsHR1X/1Xl0TMc/pHhjU1yNXzWnI/kOlNV2CRq3slEtcWihaEo8oDHJ+BhGT49Ps3Je7UB2xO/jXXFPhwJotPMOacTcnUkGqVJSlK1g6TIn4t9nTVSY8KFUs=
local_dir: dpl_cd_upload
skip_cleanup: true
region: eu-west-1
wait-until-deployed: true
on: &2
repo: alphagov/notifications-api
branch: staging
bucket: staging-notifications-api-codedeploy
- provider: codedeploy
access_key_id: AKIAJ5MKF6G3P2JQP4QQ
secret_access_key: *1
bucket: staging-notifications-api-codedeploy
key: notifications-api-$TRAVIS_BRANCH-$TRAVIS_BUILD_NUMBER-$TRAVIS_COMMIT.zip
bundle_type: zip
application: notifications-api
deployment_group: staging_delivery_api_deployment_group
region: eu-west-1
wait-until-deployed: true
on: *2
- provider: codedeploy
access_key_id: AKIAJ5MKF6G3P2JQP4QQ
secret_access_key: *1
bucket: staging-notifications-api-codedeploy
key: notifications-api-$TRAVIS_BRANCH-$TRAVIS_BUILD_NUMBER-$TRAVIS_COMMIT.zip
bundle_type: zip
application: notifications-api
deployment_group: staging_public_api_deployment_group
region: eu-west-1
wait-until-deployed: true
on: *2
- provider: codedeploy
access_key_id: AKIAJ5MKF6G3P2JQP4QQ
secret_access_key: *1
bucket: staging-notifications-api-codedeploy
key: notifications-api-$TRAVIS_BRANCH-$TRAVIS_BUILD_NUMBER-$TRAVIS_COMMIT.zip
bundle_type: zip
application: notifications-api
deployment_group: staging_admin_api_deployment_group
region: eu-west-1
wait-until-deployed: true
on: *2
- provider: s3
access_key_id: AKIAJ5MKF6G3P2JQP4QQ
secret_access_key: &1
secure: daC1bCHXqLRK+iIZ8P699KCnTh77lwV4KxrZxL1yd6cstgfptyd/rg1WgRwE6QdxOCT9gQvKWUZFCzFy7M6E/Ih8EUHqEXTzC5M4oAye8rhePIBMQwqkgfYyIoZ3LdDMMP5JfBhiz0zS3Vj7HerL2qIu12adJBjkRJx3XAGimCrFOMQ0xUXQAKDjL6Xmv+gVz2f/ISLy6icKY4KNGt3cQV+8pa5aMF34C9R2udA9N67EWlXlh7hJbFtmY+0Zqpo8Rr6wKRb5MA0xEcTVLORSz1aa6GkxUCbzaIH99p7z3Ghz0qW2bUi9ZcDrvg0GLbVe1T+1HXhfktJfW8wnzw6A/2U/CIIFDQZ/qk0w/DkEwpQinXow99Zl49CcEU+v8llKhg5nM3LmAZCQg1c/iZyP/d90AwAMoMA/VTDD72M93IqTJQH18eC8g02DwE0hNDD6aos5wzeuDeiH/6BG+Tq0pDl0y0aWCcHf3vGRlo/5GlWfpE0vMQEC+qnEOWOUqSprCdSypgD2Aip9mCC98w4BkqKKvGNHPZolA7rxf7E9hTK+BNPRATpYsHR1X/1Xl0TMc/pHhjU1yNXzWnI/kOlNV2CRq3slEtcWihaEo8oDHJ+BhGT49Ps3Je7UB2xO/jXXFPhwJotPMOacTcnUkGqVJSlK1g6TIn4t9nTVSY8KFUs=
local_dir: dpl_cd_upload
skip_cleanup: true
region: eu-west-1
wait-until-deployed: true
on: &2
repo: alphagov/notifications-api
branch: live
bucket: live-notifications-api-codedeploy
- provider: codedeploy
access_key_id: AKIAJ5MKF6G3P2JQP4QQ
secret_access_key: *1
bucket: live-notifications-api-codedeploy
key: notifications-api-$TRAVIS_BRANCH-$TRAVIS_BUILD_NUMBER-$TRAVIS_COMMIT.zip
bundle_type: zip
application: notifications-api
deployment_group: live_delivery_api_deployment_group
region: eu-west-1
wait-until-deployed: true
on: *2
- provider: codedeploy
access_key_id: AKIAJ5MKF6G3P2JQP4QQ
secret_access_key: *1
bucket: live-notifications-api-codedeploy
key: notifications-api-$TRAVIS_BRANCH-$TRAVIS_BUILD_NUMBER-$TRAVIS_COMMIT.zip
bundle_type: zip
application: notifications-api
deployment_group: live_public_api_deployment_group
region: eu-west-1
wait-until-deployed: true
on: *2
- provider: codedeploy
access_key_id: AKIAJ5MKF6G3P2JQP4QQ
secret_access_key: *1
bucket: live-notifications-api-codedeploy
key: notifications-api-$TRAVIS_BRANCH-$TRAVIS_BUILD_NUMBER-$TRAVIS_COMMIT.zip
bundle_type: zip
application: notifications-api
deployment_group: live_admin_api_deployment_group
region: eu-west-1
wait-until-deployed: true
on: *2
before_deploy:
- ./scripts/update_version_file.sh
- zip -r --exclude=*__pycache__* notifications-api *
- mkdir -p dpl_cd_upload
- mv notifications-api.zip dpl_cd_upload/notifications-api-$TRAVIS_BRANCH-$TRAVIS_BUILD_NUMBER-$TRAVIS_COMMIT.zip

View File

@@ -17,6 +17,7 @@ from app.clients.email.aws_ses import AwsSesClient
from app.clients.statsd.statsd_client import StatsdClient
from app.encryption import Encryption
DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%f"
DATE_FORMAT = "%Y-%m-%d"
@@ -38,7 +39,8 @@ api_user = LocalProxy(lambda: _request_ctx_stack.top.api_user)
def create_app(app_name=None):
application = Flask(__name__)
application.config.from_object(os.environ['NOTIFY_API_ENVIRONMENT'])
from config import configs
application.config.from_object(configs[os.environ['NOTIFY_ENVIRONMENT']])
if app_name:
application.config['NOTIFY_APP_NAME'] = app_name

View File

@@ -1,3 +1,3 @@
__travis_commit__ = "dev"
__time__ = "dev"
__travis_job_number__ = "dev"
__travis_commit__ = ""
__time__ = "2016-07-05:15:38:37"
__travis_job_number__ = ""

View File

@@ -1,14 +1,14 @@
---
files:
-
destination: /home/ubuntu/notifications-api
destination: /home/notify-app/notifications-api
source: /
hooks:
AfterInstall:
-
location: scripts/aws_install_dependencies.sh
runas: root
timeout: 300
timeout: 1000
-
location: scripts/aws_change_ownership.sh
runas: root

View File

@@ -3,19 +3,8 @@ from app import notify_celery, create_app
from credstash import getAllSecrets
import os
default_env_file = '/home/ubuntu/environment'
environment = 'live'
if os.path.isfile(default_env_file):
with open(default_env_file, 'r') as environment_file:
environment = environment_file.readline().strip()
# on aws get secrets and export to env
os.environ.update(getAllSecrets(region="eu-west-1"))
from config import configs
os.environ['NOTIFY_API_ENVIRONMENT'] = configs[environment]
application = create_app()
application.app_context().push()

View File

@@ -92,10 +92,12 @@ class Config(object):
Queue('retry', Exchange('default'), routing_key='retry'),
Queue('email-already-registered', Exchange('default'), routing_key='email-already-registered')
]
API_HOST_NAME = os.environ['API_HOST_NAME']
MMG_API_KEY = os.environ['MMG_API_KEY']
FIRETEXT_API_KEY = os.getenv("FIRETEXT_API_KEY")
LOADTESTING_NUMBER = os.getenv('LOADTESTING_NUMBER')
LOADTESTING_API_KEY = os.getenv("LOADTESTING_API_KEY")
CSV_UPLOAD_BUCKET_NAME = 'local-notifications-csv-upload'
CSV_UPLOAD_BUCKET_NAME = os.getenv("CSV_UPLOAD_BUCKET_NAME")
NOTIFICATIONS_ALERT = 5 # five mins
FROM_NUMBER = os.getenv('FROM_NUMBER')
@@ -109,27 +111,29 @@ class Config(object):
class Development(Config):
DEBUG = True
API_HOST_NAME = os.environ['API_HOST_NAME']
MMG_API_KEY = os.environ['MMG_API_KEY']
CSV_UPLOAD_BUCKET_NAME = 'development-notifications-csv-upload'
class Preview(Config):
MMG_API_KEY = os.environ['MMG_API_KEY']
API_HOST_NAME = os.environ['API_HOST_NAME']
CSV_UPLOAD_BUCKET_NAME = 'preview-notifications-csv-upload'
CSV_UPLOAD_BUCKET_NAME = os.getenv("CSV_UPLOAD_BUCKET_NAME")
class Test(Development):
MMG_API_KEY = os.environ['MMG_API_KEY']
API_HOST_NAME = os.environ['API_HOST_NAME']
CSV_UPLOAD_BUCKET_NAME = 'test-notifications-csv-upload'
class Staging(Config):
CSV_UPLOAD_BUCKET_NAME = 'staging-notify-csv-upload'
class Live(Config):
CSV_UPLOAD_BUCKET_NAME = 'live-notifications-csv-upload'
STATSD_ENABLED = True
configs = {
'development': 'config.Development',
'test': 'config.Test',
'live': 'config_live.Live',
'staging': 'config_staging.Staging',
'preview': 'config.Preview'
'development': Development,
'test': Test,
'live': Live,
'staging': Staging,
'preview': Preview
}

View File

@@ -1,30 +0,0 @@
import os
from config import Config
class Live(Config):
ADMIN_BASE_URL = os.environ['LIVE_ADMIN_BASE_URL']
API_HOST_NAME = os.environ['LIVE_API_HOST_NAME']
ADMIN_CLIENT_SECRET = os.environ['LIVE_ADMIN_CLIENT_SECRET']
DANGEROUS_SALT = os.environ['LIVE_DANGEROUS_SALT']
NOTIFICATION_QUEUE_PREFIX = os.environ['LIVE_NOTIFICATION_QUEUE_PREFIX']
NOTIFY_JOB_QUEUE = os.environ['LIVE_NOTIFY_JOB_QUEUE']
SECRET_KEY = os.environ['LIVE_SECRET_KEY']
SQLALCHEMY_DATABASE_URI = os.environ['LIVE_SQLALCHEMY_DATABASE_URI']
VERIFY_CODE_FROM_EMAIL_ADDRESS = os.environ['LIVE_VERIFY_CODE_FROM_EMAIL_ADDRESS']
NOTIFY_EMAIL_DOMAIN = os.environ['LIVE_NOTIFY_EMAIL_DOMAIN']
FIRETEXT_API_KEY = os.getenv("LIVE_FIRETEXT_API_KEY")
MMG_API_KEY = os.environ['LIVE_MMG_API_KEY']
CSV_UPLOAD_BUCKET_NAME = 'live-notifications-csv-upload'
STATSD_ENABLED = True
STATSD_HOST = os.getenv('LIVE_STATSD_HOST')
STATSD_PORT = os.getenv('LIVE_STATSD_PORT')
STATSD_PREFIX = os.getenv('LIVE_STATSD_PREFIX')
FROM_NUMBER = os.getenv('LIVE_FROM_NUMBER')
BROKER_TRANSPORT_OPTIONS = {
'region': 'eu-west-1',
'polling_interval': 1, # 1 second
'visibility_timeout': 14410, # 60 seconds
'queue_name_prefix': os.environ['LIVE_NOTIFICATION_QUEUE_PREFIX'] + '-'
}

View File

@@ -1,26 +0,0 @@
import os
from config import Config
class Staging(Config):
ADMIN_BASE_URL = os.environ['STAGING_ADMIN_BASE_URL']
API_HOST_NAME = os.environ['STAGING_API_HOST_NAME']
ADMIN_CLIENT_SECRET = os.environ['STAGING_ADMIN_CLIENT_SECRET']
DANGEROUS_SALT = os.environ['STAGING_DANGEROUS_SALT']
NOTIFICATION_QUEUE_PREFIX = os.environ['STAGING_NOTIFICATION_QUEUE_PREFIX']
NOTIFY_JOB_QUEUE = os.environ['STAGING_NOTIFY_JOB_QUEUE']
SECRET_KEY = os.environ['STAGING_SECRET_KEY']
SQLALCHEMY_DATABASE_URI = os.environ['STAGING_SQLALCHEMY_DATABASE_URI']
VERIFY_CODE_FROM_EMAIL_ADDRESS = os.environ['STAGING_VERIFY_CODE_FROM_EMAIL_ADDRESS']
NOTIFY_EMAIL_DOMAIN = os.environ['STAGING_NOTIFY_EMAIL_DOMAIN']
FIRETEXT_API_KEY = os.getenv("STAGING_FIRETEXT_API_KEY")
MMG_API_KEY = os.environ['STAGING_MMG_API_KEY']
CSV_UPLOAD_BUCKET_NAME = 'staging-notifications-csv-upload'
FROM_NUMBER = os.getenv('STAGING_FROM_NUMBER')
BROKER_TRANSPORT_OPTIONS = {
'region': 'eu-west-1',
'polling_interval': 1, # 1 second
'visibility_timeout': 14410, # 60 seconds
'queue_name_prefix': os.environ['STAGING_NOTIFICATION_QUEUE_PREFIX'] + '-'
}

19
db.py
View File

@@ -4,19 +4,18 @@ from app import create_app, db
from credstash import getAllSecrets
import os
default_env_file = '/home/ubuntu/environment'
environment = 'live'
if os.path.isfile(default_env_file):
with open(default_env_file, 'r') as environment_file:
environment = environment_file.readline().strip()
# on aws get secrets and export to env
os.environ.update(getAllSecrets(region="eu-west-1"))
from config import configs
os.environ['NOTIFY_API_ENVIRONMENT'] = configs[environment]
print("DOING SETUP")
print("\n" * 10)
print("SECRETS")
print("\n" * 10)
print(getAllSecrets(region="eu-west-1"))
print("\n" * 10)
print("ENV")
print("\n" * 10)
print(os.environ)
application = create_app()

View File

@@ -1,5 +1,5 @@
#!/bin/bash
export NOTIFY_API_ENVIRONMENT='config.Test'
export NOTIFY_ENVIRONMENT='test'
export ADMIN_BASE_URL='http://localhost:6012'
export ADMIN_CLIENT_USER_NAME='dev-notify-admin'
export ADMIN_CLIENT_SECRET='dev-notify-secret-key'

View File

@@ -1,5 +1,13 @@
#!/bin/bash
echo "Chown application to be owned by ubuntu"
cd /home/ubuntu/;
chown -R ubuntu:ubuntu notifications-api
if [ -e "/home/notify-app" ]
then
echo "Chown application to be owned by notify-app"
cd /home/notify-app/;
chown -R notify-app:govuk-notify-applications notifications-api
else
echo "Chown application to be owned by ubuntu"
cd /home/ubuntu/;
chown -R ubuntu:ubuntu notifications-api
fi

View File

@@ -1,4 +1,15 @@
echo "Install dependencies"
cd /home/ubuntu/notifications-api;
pip3 install -r /home/ubuntu/notifications-api/requirements.txt
python3 db.py db upgrade
if [ -e "/home/notify-app" ]
then
echo "Depenencies for notify-app"
cd /home/notify-app/notifications-api;
pip3 install -r /home/notify-app/notifications-api/requirements.txt
python3 db.py db upgrade
else
echo "Depenencies for ubuntu"
cd /home/ubuntu/notifications-api;
pip3 install -r /home/ubuntu/notifications-api/requirements.txt
python3 db.py db upgrade
fi

View File

@@ -101,7 +101,7 @@ reset_waiter_timeout() {
fi
# Base register/deregister action may take up to about 30 seconds
timeout=$((timeout + 30))
timeout=$((timeout + 60))
WAITER_ATTEMPTS=$((timeout / WAITER_INTERVAL))
}
@@ -371,7 +371,9 @@ get_elb_name_for_instance_name() {
local instance_name=$1
declare -A elb_to_instance_mapping
elb_to_instance_mapping['notify_api']='notify-api-elb'
elb_to_instance_mapping['notify_admin_api']='notify-admin-api-elb'
elb_to_instance_mapping['live_notify_api']='live-notify-api-elb'
elb_to_instance_mapping['staging_notify_api']='staging-notify-api-elb'
elb_to_instance_mapping['NotifyApi']='notify-api-elb'

11
wsgi.py
View File

@@ -4,20 +4,9 @@ from app import create_app
from credstash import getAllSecrets
default_env_file = '/home/ubuntu/environment'
environment = 'live'
if os.path.isfile(default_env_file):
with open(default_env_file, 'r') as environment_file:
environment = environment_file.readline().strip()
# on aws get secrets and export to env
os.environ.update(getAllSecrets(region="eu-west-1"))
from config import configs
os.environ['NOTIFY_API_ENVIRONMENT'] = configs[environment]
application = create_app()
if __name__ == "__main__":