From 9c212e78af2b0e2c17b4375581226660b93f9837 Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Mon, 31 Jul 2017 11:11:00 +0100 Subject: [PATCH 1/3] Revert "Revert "remove credstash"" This reverts commit de41fde0d37e3f52b316bcda74445c07d04eb28b. --- aws_run_celery.py | 8 +------- db.py | 8 ++------ requirements.txt | 1 - server_commands.py | 5 ----- wsgi.py | 7 ------- 5 files changed, 3 insertions(+), 26 deletions(-) diff --git a/aws_run_celery.py b/aws_run_celery.py index 36a1f480a..8194528db 100644 --- a/aws_run_celery.py +++ b/aws_run_celery.py @@ -1,11 +1,5 @@ #!/usr/bin/env python -from app import notify_celery, create_app -from credstash import getAllSecrets -import os - -# On AWS get secrets and export to env, skip this on Cloud Foundry -if os.getenv('VCAP_SERVICES') is None: - os.environ.update(getAllSecrets(region="eu-west-1")) +from app import create_app application = create_app("delivery") application.app_context().push() diff --git a/db.py b/db.py index 3a8da01d4..bc558cbd2 100644 --- a/db.py +++ b/db.py @@ -1,12 +1,8 @@ from flask.ext.script import Manager, Server from flask_migrate import Migrate, MigrateCommand -from app import create_app, db -from credstash import getAllSecrets -import os -# On AWS get secrets and export to env, skip this on Cloud Foundry -if os.getenv('VCAP_SERVICES') is None: - os.environ.update(getAllSecrets(region="eu-west-1")) +from app import create_app, db + application = create_app() diff --git a/requirements.txt b/requirements.txt index 0c9b3f1e0..61390b6c4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,6 @@ marshmallow==2.4.2 marshmallow-sqlalchemy==0.8.0 flask-marshmallow==0.6.2 Flask-Bcrypt==0.6.2 -credstash==1.8.0 boto3==1.4.4 celery==3.1.25 # pyup: <4 monotonic==1.2 diff --git a/server_commands.py b/server_commands.py index 85bf13137..af071db47 100644 --- a/server_commands.py +++ b/server_commands.py @@ -1,7 +1,6 @@ from flask.ext.script import Manager, Server from flask_migrate import Migrate, MigrateCommand from app import (create_app, db, commands) -from credstash import getAllSecrets import os default_env_file = '/home/ubuntu/environment' @@ -11,10 +10,6 @@ if os.path.isfile(default_env_file): with open(default_env_file, 'r') as environment_file: environment = environment_file.readline().strip() -# On AWS get secrets and export to env, skip this on Cloud Foundry -if os.getenv('VCAP_SERVICES') is None: - os.environ.update(getAllSecrets(region="eu-west-1")) - from app.config import configs os.environ['NOTIFY_API_ENVIRONMENT'] = configs[environment] diff --git a/wsgi.py b/wsgi.py index 2df2c3976..9fbeb28ac 100644 --- a/wsgi.py +++ b/wsgi.py @@ -1,13 +1,6 @@ -import os - from app import create_app -from credstash import getAllSecrets -# On AWS get secrets and export to env, skip this on Cloud Foundry -if os.getenv('VCAP_SERVICES') is None: - os.environ.update(getAllSecrets(region="eu-west-1")) - application = create_app() if __name__ == "__main__": From b5dc7642aa574b60e172142ad88fd0e42d56528d Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Mon, 31 Jul 2017 11:12:43 +0100 Subject: [PATCH 2/3] remove aws_run_celery file it's no longer relevant since the switch to PaaS --- aws_run_celery.py | 5 ----- manifest-delivery-base.yml | 14 +++++++------- run_celery.py | 1 + 3 files changed, 8 insertions(+), 12 deletions(-) delete mode 100644 aws_run_celery.py diff --git a/aws_run_celery.py b/aws_run_celery.py deleted file mode 100644 index 8194528db..000000000 --- a/aws_run_celery.py +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env python -from app import create_app - -application = create_app("delivery") -application.app_context().push() diff --git a/manifest-delivery-base.yml b/manifest-delivery-base.yml index 597df9f3f..6fba774ea 100644 --- a/manifest-delivery-base.yml +++ b/manifest-delivery-base.yml @@ -16,39 +16,39 @@ memory: 1G applications: - name: notify-delivery-celery-beat - command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery beat --loglevel=INFO + command: scripts/run_app_paas.sh celery -A run_celery.notify_celery beat --loglevel=INFO instances: 1 memory: 128M env: NOTIFY_APP_NAME: delivery-celery-beat - name: notify-delivery-worker-database - command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q database-tasks + command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q database-tasks env: NOTIFY_APP_NAME: delivery-worker-database - name: notify-delivery-worker-research - command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=5 -Q research-mode-tasks + command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=5 -Q research-mode-tasks env: NOTIFY_APP_NAME: delivery-worker-research - name: notify-delivery-worker-sender - command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q send-tasks,send-sms-tasks,send-email-tasks + command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q send-tasks,send-sms-tasks,send-email-tasks env: NOTIFY_APP_NAME: delivery-worker-sender - name: notify-delivery-worker-periodic - command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=2 -Q periodic-tasks,statistics-tasks + command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=2 -Q periodic-tasks,statistics-tasks instances: 1 env: NOTIFY_APP_NAME: delivery-worker-periodic - name: notify-delivery-worker-priority - command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=5 -Q priority-tasks + command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=5 -Q priority-tasks env: NOTIFY_APP_NAME: delivery-worker-priority - name: notify-delivery-worker - command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q job-tasks,retry-tasks,notify-internal-tasks + command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q job-tasks,retry-tasks,notify-internal-tasks env: NOTIFY_APP_NAME: delivery-worker diff --git a/run_celery.py b/run_celery.py index 066735e63..013499615 100644 --- a/run_celery.py +++ b/run_celery.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +# notify_celery is referenced from manifest_delivery_base.yml, and cannot be removed from app import notify_celery, create_app application = create_app('delivery') From 3cb3cf438ec905d7050572d8804310b6b67db3f9 Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Mon, 31 Jul 2017 13:29:30 +0100 Subject: [PATCH 3/3] remove SEND_COMBINED --- app/config.py | 2 -- manifest-delivery-base.yml | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/app/config.py b/app/config.py index 5d71c0f41..cfd37fa33 100644 --- a/app/config.py +++ b/app/config.py @@ -22,7 +22,6 @@ class QueueNames(object): PERIODIC = 'periodic-tasks' PRIORITY = 'priority-tasks' DATABASE = 'database-tasks' - SEND_COMBINED = 'send-tasks' SEND_SMS = 'send-sms-tasks' SEND_EMAIL = 'send-email-tasks' RESEARCH_MODE = 'research-mode-tasks' @@ -38,7 +37,6 @@ class QueueNames(object): QueueNames.PRIORITY, QueueNames.PERIODIC, QueueNames.DATABASE, - QueueNames.SEND_COMBINED, QueueNames.SEND_SMS, QueueNames.SEND_EMAIL, QueueNames.RESEARCH_MODE, diff --git a/manifest-delivery-base.yml b/manifest-delivery-base.yml index 6fba774ea..78053a729 100644 --- a/manifest-delivery-base.yml +++ b/manifest-delivery-base.yml @@ -33,7 +33,7 @@ applications: NOTIFY_APP_NAME: delivery-worker-research - name: notify-delivery-worker-sender - command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q send-tasks,send-sms-tasks,send-email-tasks + command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q send-sms-tasks,send-email-tasks env: NOTIFY_APP_NAME: delivery-worker-sender