From fb41acdac9bf26c1cc9e443088c71b1900246e4d Mon Sep 17 00:00:00 2001 From: Martyn Inglis Date: Tue, 9 Feb 2016 13:31:45 +0000 Subject: [PATCH 01/17] Celery tests --- app/__init__.py | 6 +++++- app/celery/__init__.py | 0 app/celery/celery.py | 21 +++++++++++++++++++++ app/celery/tasks.py | 9 +++++++++ app/notifications/rest.py | 16 ++++++++++++++-- config.py | 24 ++++++++++++++++++++++++ requirements.txt | 2 ++ run_celery.py | 7 +++++++ wsgi.py | 4 ++-- 9 files changed, 84 insertions(+), 5 deletions(-) create mode 100644 app/celery/__init__.py create mode 100644 app/celery/celery.py create mode 100644 app/celery/tasks.py create mode 100644 run_celery.py diff --git a/app/__init__.py b/app/__init__.py index 5fefc46d4..ad1f44056 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -11,12 +11,13 @@ from werkzeug.local import LocalProxy from config import configs from utils import logging from notify_client import NotifyAPIClient +from app.celery.celery import NotifyCelery db = SQLAlchemy() ma = Marshmallow() notify_alpha_client = NotifyAPIClient() - +celery = NotifyCelery() api_user = LocalProxy(lambda: _request_ctx_stack.top.api_user) @@ -32,6 +33,8 @@ def create_app(config_name, config_overrides=None): logging.init_app(application) notify_alpha_client.init_app(application) + celery.init_app(application) + from app.service.rest import service as service_blueprint from app.user.rest import user as user_blueprint from app.template.rest import template as template_blueprint @@ -75,6 +78,7 @@ def init_app(app, config_overrides): return response + def convert_to_boolean(value): """Turn strings to bools if they look like them diff --git a/app/celery/__init__.py b/app/celery/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/app/celery/celery.py b/app/celery/celery.py new file mode 100644 index 000000000..c98d4367c --- /dev/null +++ b/app/celery/celery.py @@ -0,0 +1,21 @@ +from celery import Celery + + +class NotifyCelery(Celery): + + def init_app(self, app): + super().__init__(app.import_name, broker=app.config['BROKER_URL']) + self.conf.update(app.config) + TaskBase = self.Task + + class ContextTask(TaskBase): + abstract = True + + def __call__(self, *args, **kwargs): + with app.app_context(): + return TaskBase.__call__(self, *args, **kwargs) + self.Task = ContextTask + + + + diff --git a/app/celery/tasks.py b/app/celery/tasks.py new file mode 100644 index 000000000..736ddeb79 --- /dev/null +++ b/app/celery/tasks.py @@ -0,0 +1,9 @@ +from app import celery +from app.dao.services_dao import get_model_services + + +@celery.task(name="refresh-services") +def refresh_services(): + print(get_model_services()) + for service in get_model_services(): + celery.control.add_consumer(str(service.id)) diff --git a/app/notifications/rest.py b/app/notifications/rest.py index a130ade45..1fe16e338 100644 --- a/app/notifications/rest.py +++ b/app/notifications/rest.py @@ -12,6 +12,7 @@ from app import notify_alpha_client from app import api_user from app.dao import (templates_dao, services_dao) import re +from app import celery mobile_regex = re.compile("^\\+44[\\d]{10}$") @@ -23,6 +24,15 @@ def get_notifications(notification_id): return jsonify(notify_alpha_client.fetch_notification_by_id(notification_id)), 200 +@celery.task(name="make-sms", bind="True") +def send_sms(self, to, template): + print('Executing task id {0.id}, args: {0.args!r} kwargs: {0.kwargs!r}'.format(self.request)) + from time import sleep + sleep(0.5) + print('finished') + #notify_alpha_client.send_sms(mobile_number=to, message=template) + + @notifications.route('/sms', methods=['POST']) def create_sms_notification(): notification = request.get_json()['notification'] @@ -54,8 +64,9 @@ def create_sms_notification(): # add notification to the queue service = services_dao.get_model_services(api_user['client'], _raise=False) - _add_notification_to_queue(template.id, service, 'sms', to) - return jsonify(notify_alpha_client.send_sms(mobile_number=to, message=template.content)), 200 + #_add_notification_to_queue(template.id, service, 'sms', to) + send_sms.apply_async((to, template.content), queue=str(service.id)) + return jsonify(success=True) # notify_alpha_client.send_sms(mobile_number=to, message=template.content)), 200 @notifications.route('/email', methods=['POST']) @@ -151,3 +162,4 @@ def _add_notification_to_queue(template_id, service, msg_type, to): 'message_id': {'StringValue': message_id, 'DataType': 'String'}, 'service_id': {'StringValue': str(service.id), 'DataType': 'String'}, 'template_id': {'StringValue': str(template_id), 'DataType': 'String'}}) + diff --git a/config.py b/config.py index 14d595348..eaa62e0ac 100644 --- a/config.py +++ b/config.py @@ -1,4 +1,5 @@ import os +from datetime import timedelta class Config(object): @@ -17,6 +18,25 @@ class Config(object): AWS_REGION = 'eu-west-1' NOTIFY_JOB_QUEUE = os.getenv('NOTIFY_JOB_QUEUE', 'notify-jobs-queue') + BROKER_URL = 'amqp://guest:guest@localhost:5672//' + BROKER_TRANSPORT_OPTIONS = { + 'region': 'eu-west-1', + 'polling_interval': 10, # 1 second + 'visibility_timeout': 3600, # 1 hour + 'queue_name_prefix': 'NOTIFY-CELERY-TEST-' + } + CELERY_ENABLE_UTC = True, + CELERY_TIMEZONE = 'Europe/London' + CELERY_ACCEPT_CONTENT = ['json'] + CELERY_TASK_SERIALIZER = 'json' + CELERYBEAT_SCHEDULE = { + 'refresh-queues': { + 'task': 'refresh-services', + 'schedule': timedelta(seconds=5) + } + } + CELERY_IMPORTS = ('app.celery.tasks',) + class Development(Config): DEBUG = True @@ -36,6 +56,10 @@ class Test(Config): class Live(Config): + SECRET_KEY = 'secret-key' + DANGEROUS_SALT = 'dangerous-salt' + ADMIN_CLIENT_USER_NAME = 'dev-notify-admin' + ADMIN_CLIENT_SECRET = 'dev-notify-secret-key' pass diff --git a/requirements.txt b/requirements.txt index 96d3663b6..2de23f16b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,6 +13,8 @@ itsdangerous==0.24 Flask-Bcrypt==0.6.2 credstash==1.8.0 boto3==1.2.3 +celery==3.1.20 +redis==2.10.5 git+https://github.com/alphagov/notifications-python-client.git@0.2.1#egg=notifications-python-client==0.2.1 diff --git a/run_celery.py b/run_celery.py new file mode 100644 index 000000000..e4eb8eb0a --- /dev/null +++ b/run_celery.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python +import os +from app import celery, create_app +from app.celery.tasks import refresh_services + +application = create_app(os.getenv('NOTIFY_API_ENVIRONMENT') or 'development') +application.app_context().push() diff --git a/wsgi.py b/wsgi.py index 138a48c6f..6e68a6c86 100644 --- a/wsgi.py +++ b/wsgi.py @@ -1,9 +1,9 @@ from app import create_app from credstash import getAllSecrets -secrets = getAllSecrets(region="eu-west-1") +#secrets = getAllSecrets(region="eu-west-1") -application = create_app('live', secrets) +application = create_app('live', None) if __name__ == "__main__": application.run() From 16c5e7bf10f4f96a8fdde56e479dbceabae36e83 Mon Sep 17 00:00:00 2001 From: Martyn Inglis Date: Mon, 15 Feb 2016 11:11:20 +0000 Subject: [PATCH 02/17] Celery task added --- app/notifications/rest.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/app/notifications/rest.py b/app/notifications/rest.py index 0c3cdf569..aedbc79ff 100644 --- a/app/notifications/rest.py +++ b/app/notifications/rest.py @@ -11,6 +11,7 @@ from app.aws_sqs import add_notification_to_queue from app.dao import (templates_dao) from app.schemas import ( email_notification_schema, sms_template_notification_schema) +from app import celery notifications = Blueprint('notifications', __name__) @@ -21,6 +22,14 @@ def get_notifications(notification_id): return jsonify({'id': notification_id}), 200 +@celery.task(name="make-sms", bind="True") +def send_sms(self): + print('Executing task id {0.id}, args: {0.args!r} kwargs: {0.kwargs!r}'.format(self.request)) + from time import sleep + sleep(0.5) + print('finished') + + @notifications.route('/sms', methods=['POST']) def create_sms_notification(): resp_json = request.get_json() @@ -29,6 +38,7 @@ def create_sms_notification(): if errors: return jsonify(result="error", message=errors), 400 + send_sms.delay() add_notification_to_queue(api_user['client'], notification['template'], 'sms', notification) # TODO data to be returned return jsonify({}), 204 From 223cb8c2dd1b3c08f5a1752f05b3fa4aed020ff7 Mon Sep 17 00:00:00 2001 From: Martyn Inglis Date: Mon, 15 Feb 2016 16:01:14 +0000 Subject: [PATCH 03/17] Made SMS messages go through celery - twilio client pulled in from delivery app - made method to perform task --- app/__init__.py | 6 +++-- app/celery/tasks.py | 38 +++++++++++++++++++++++------ app/clients/__init__.py | 13 ++++++++++ app/clients/sms/__init__.py | 17 +++++++++++++ app/clients/sms/twilio.py | 48 +++++++++++++++++++++++++++++++++++++ app/notifications/rest.py | 33 +++++++++++++++---------- config.py | 17 +++++++------ requirements.txt | 3 ++- run_celery.py | 1 - 9 files changed, 145 insertions(+), 31 deletions(-) create mode 100644 app/clients/__init__.py create mode 100644 app/clients/sms/__init__.py create mode 100644 app/clients/sms/twilio.py diff --git a/app/__init__.py b/app/__init__.py index e64a1b51e..619824c16 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -12,12 +12,14 @@ from config import configs from utils import logging from notify_client import NotifyAPIClient from app.celery.celery import NotifyCelery - +from app.clients.sms.twilio import TwilioClient db = SQLAlchemy() ma = Marshmallow() notify_alpha_client = NotifyAPIClient() celery = NotifyCelery() +twilio_client = TwilioClient() + api_user = LocalProxy(lambda: _request_ctx_stack.top.api_user) @@ -32,7 +34,7 @@ def create_app(config_name, config_overrides=None): ma.init_app(application) init_app(application, config_overrides) logging.init_app(application) - + twilio_client.init_app(application) celery.init_app(application) from app.service.rest import service as service_blueprint diff --git a/app/celery/tasks.py b/app/celery/tasks.py index 736ddeb79..e8d20d009 100644 --- a/app/celery/tasks.py +++ b/app/celery/tasks.py @@ -1,9 +1,33 @@ -from app import celery -from app.dao.services_dao import get_model_services +from itsdangerous import URLSafeSerializer +from app import celery, twilio_client, db +from app.clients.sms.twilio import TwilioClientException +from app.dao.templates_dao import get_model_templates +from app.models import Notification +from flask import current_app -@celery.task(name="refresh-services") -def refresh_services(): - print(get_model_services()) - for service in get_model_services(): - celery.control.add_consumer(str(service.id)) +@celery.task(name="send-sms", bind="True") +def send_sms(service_id, notification_id, encrypted_notification, secret_key, salt): + serializer = URLSafeSerializer(secret_key) + + notification = serializer.loads(encrypted_notification, salt=salt) + template = get_model_templates(notification['template']) + + status = 'sent' + + try: + twilio_client.send_sms(notification, template.content) + except TwilioClientException as e: + current_app.logger.info(e) + status = 'failed' + + notification_db_object = Notification( + id=notification_id, + template_id=notification['template'], + to=notification['to'], + service_id=service_id, + status=status + ) + + db.session.add(notification_db_object) + db.session.commit() diff --git a/app/clients/__init__.py b/app/clients/__init__.py new file mode 100644 index 000000000..06495ccbe --- /dev/null +++ b/app/clients/__init__.py @@ -0,0 +1,13 @@ + +class ClientException(Exception): + ''' + Base Exceptions for sending notifications that fail + ''' + pass + + +class Client(object): + ''' + Base client for sending notifications. + ''' + pass diff --git a/app/clients/sms/__init__.py b/app/clients/sms/__init__.py new file mode 100644 index 000000000..8b83ab8b9 --- /dev/null +++ b/app/clients/sms/__init__.py @@ -0,0 +1,17 @@ +from app.clients import (Client, ClientException) + + +class SmsClientException(ClientException): + ''' + Base Exception for SmsClients + ''' + pass + + +class SmsClient(Client): + ''' + Base Sms client for sending smss. + ''' + + def send_sms(self, *args, **kwargs): + raise NotImplemented('TODO Need to implement.') diff --git a/app/clients/sms/twilio.py b/app/clients/sms/twilio.py new file mode 100644 index 000000000..0a3eaee69 --- /dev/null +++ b/app/clients/sms/twilio.py @@ -0,0 +1,48 @@ +import logging +from app.clients.sms import ( + SmsClient, SmsClientException) +from twilio.rest import TwilioRestClient +from twilio import TwilioRestException + + +logger = logging.getLogger(__name__) + + +class TwilioClientException(SmsClientException): + pass + + +class TwilioClient(SmsClient): + ''' + Twilio sms client. + ''' + def init_app(self, config, *args, **kwargs): + super(TwilioClient, self).__init__(*args, **kwargs) + self.client = TwilioRestClient( + config.config.get('TWILIO_ACCOUNT_SID'), + config.config.get('TWILIO_AUTH_TOKEN')) + self.from_number = config.config.get('TWILIO_NUMBER') + print(config.config) + + + def send_sms(self, notification, content): + try: + response = self.client.messages.create( + body=content, + to=notification['to'], + from_=self.from_number + ) + return response.sid + except TwilioRestException as e: + logger.exception(e) + raise TwilioClientException(e) + + def status(self, message_id): + try: + response = self.client.messages.get(message_id) + if response.status in ('delivered', 'undelivered', 'failed'): + return response.status + return None + except TwilioRestException as e: + logger.exception(e) + raise TwilioClientException(e) diff --git a/app/notifications/rest.py b/app/notifications/rest.py index 6effdb8eb..8bf50002e 100644 --- a/app/notifications/rest.py +++ b/app/notifications/rest.py @@ -3,43 +3,50 @@ import uuid from flask import ( Blueprint, jsonify, - request -) + request, + current_app) +from itsdangerous import URLSafeSerializer from app import api_user from app.aws_sqs import add_notification_to_queue from app.dao import (templates_dao) from app.schemas import ( email_notification_schema, sms_template_notification_schema) -from app import celery +from app.celery.tasks import send_sms + notifications = Blueprint('notifications', __name__) +def create_notification_id(): + return str(uuid.uuid4()) + + @notifications.route('/', methods=['GET']) def get_notifications(notification_id): # TODO return notification id details return jsonify({'id': notification_id}), 200 -@celery.task(name="make-sms", bind="True") -def send_sms(self): - print('Executing task id {0.id}, args: {0.args!r} kwargs: {0.kwargs!r}'.format(self.request)) - from time import sleep - sleep(0.5) - print('finished') - - @notifications.route('/sms', methods=['POST']) def create_sms_notification(): + serializer = URLSafeSerializer(current_app.config.get('SECRET_KEY')) + resp_json = request.get_json() notification, errors = sms_template_notification_schema.load(resp_json) if errors: return jsonify(result="error", message=errors), 400 - send_sms.delay() - notification_id = add_notification_to_queue(api_user['client'], notification['template'], 'sms', notification) + notification_id = create_notification_id() + encrypted_notification = serializer.dumps(notification, current_app.config.get('DANGEROUS_SALT')) + + send_sms.apply_async(( + api_user['client'], + notification_id, + encrypted_notification, + current_app.config.get('SECRET_KEY'), + current_app.config.get('DANGEROUS_SALT'))) return jsonify({'notification_id': notification_id}), 201 diff --git a/config.py b/config.py index 61ad93ee5..b1861d8a7 100644 --- a/config.py +++ b/config.py @@ -22,7 +22,7 @@ class Config(object): # Notification Queue names are a combination of a prefx plus a name NOTIFICATION_QUEUE_PREFIX = 'notification' - BROKER_URL = 'amqp://guest:guest@localhost:5672//' + BROKER_URL = 'sqs://' BROKER_TRANSPORT_OPTIONS = { 'region': 'eu-west-1', 'polling_interval': 10, # 1 second @@ -33,13 +33,16 @@ class Config(object): CELERY_TIMEZONE = 'Europe/London' CELERY_ACCEPT_CONTENT = ['json'] CELERY_TASK_SERIALIZER = 'json' - CELERYBEAT_SCHEDULE = { - 'refresh-queues': { - 'task': 'refresh-services', - 'schedule': timedelta(seconds=5) - } - } + # CELERYBEAT_SCHEDULE = { + # 'refresh-queues': { + # 'task': 'refresh-services', + # 'schedule': timedelta(seconds=5) + # } + # } CELERY_IMPORTS = ('app.celery.tasks',) + TWILIO_ACCOUNT_SID = os.getenv('TWILIO_ACCOUNT_SID') + TWILIO_AUTH_TOKEN = os.getenv('TWILIO_AUTH_TOKEN') + TWILIO_NUMBER = os.getenv('TWILIO_NUMBER') class Development(Config): diff --git a/requirements.txt b/requirements.txt index c36305c94..184d57249 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,8 @@ Flask-Bcrypt==0.6.2 credstash==1.8.0 boto3==1.2.3 celery==3.1.20 -redis==2.10.5 +twilio==4.6.0 + git+https://github.com/alphagov/notifications-python-client.git@0.2.6#egg=notifications-python-client==0.2.6 diff --git a/run_celery.py b/run_celery.py index e4eb8eb0a..269142d5b 100644 --- a/run_celery.py +++ b/run_celery.py @@ -1,7 +1,6 @@ #!/usr/bin/env python import os from app import celery, create_app -from app.celery.tasks import refresh_services application = create_app(os.getenv('NOTIFY_API_ENVIRONMENT') or 'development') application.app_context().push() From 655beddba6e8f9ddb7073b7aa751f2e03f5f3626 Mon Sep 17 00:00:00 2001 From: Martyn Inglis Date: Tue, 16 Feb 2016 11:22:44 +0000 Subject: [PATCH 04/17] Fixed up the get_notitication endpoint - returns a notification --- app/__init__.py | 2 +- app/clients/sms/twilio.py | 2 -- app/dao/notifications_dao.py | 8 ++++++-- app/job/rest.py | 16 ++++++--------- app/notifications/rest.py | 23 ++++++++++++++-------- app/schemas.py | 2 +- tests/app/dao/test_notification_dao.py | 19 +++++++++++++----- tests/app/notifications/test_rest.py | 27 ++++++++++++++------------ 8 files changed, 58 insertions(+), 41 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index 619824c16..f035df8ed 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -30,9 +30,9 @@ def create_app(config_name, config_overrides=None): application.config['NOTIFY_API_ENVIRONMENT'] = config_name application.config.from_object(configs[config_name]) + init_app(application, config_overrides) db.init_app(application) ma.init_app(application) - init_app(application, config_overrides) logging.init_app(application) twilio_client.init_app(application) celery.init_app(application) diff --git a/app/clients/sms/twilio.py b/app/clients/sms/twilio.py index 0a3eaee69..52b0375ee 100644 --- a/app/clients/sms/twilio.py +++ b/app/clients/sms/twilio.py @@ -22,8 +22,6 @@ class TwilioClient(SmsClient): config.config.get('TWILIO_ACCOUNT_SID'), config.config.get('TWILIO_AUTH_TOKEN')) self.from_number = config.config.get('TWILIO_NUMBER') - print(config.config) - def send_sms(self, notification, content): try: diff --git a/app/dao/notifications_dao.py b/app/dao/notifications_dao.py index ed3cd0f6d..f917f4429 100644 --- a/app/dao/notifications_dao.py +++ b/app/dao/notifications_dao.py @@ -14,9 +14,13 @@ def save_notification(notification, update_dict={}): db.session.commit() -def get_notification(service_id, job_id, notification_id): +def get_notification_for_job(service_id, job_id, notification_id): return Notification.query.filter_by(service_id=service_id, job_id=job_id, id=notification_id).one() -def get_notifications(service_id, job_id): +def get_notifications_for_job(service_id, job_id): return Notification.query.filter_by(service_id=service_id, job_id=job_id).all() + + +def get_notification(service_id, notification_id): + return Notification.query.filter_by(service_id=service_id, id=notification_id).one() diff --git a/app/job/rest.py b/app/job/rest.py index 17f1e341c..1928e5d66 100644 --- a/app/job/rest.py +++ b/app/job/rest.py @@ -17,11 +17,7 @@ from app.dao.jobs_dao import ( get_jobs_by_service ) -from app.dao.notifications_dao import ( - save_notification, - get_notification, - get_notifications -) +from app.dao import notifications_dao from app.schemas import ( job_schema, @@ -89,7 +85,7 @@ def create_notification_for_job(service_id, job_id): if errors: return jsonify(result="error", message=errors), 400 try: - save_notification(notification) + notifications_dao.save_notification(notification) except Exception as e: return jsonify(result="error", message=str(e)), 500 return jsonify(data=notification_status_schema.dump(notification).data), 201 @@ -100,7 +96,7 @@ def create_notification_for_job(service_id, job_id): def get_notification_for_job(service_id, job_id, notification_id=None): if notification_id: try: - notification = get_notification(service_id, job_id, notification_id) + notification = notifications_dao.get_notification_for_job(service_id, job_id, notification_id) data, errors = notification_status_schema.dump(notification) return jsonify(data=data) except DataError: @@ -108,7 +104,7 @@ def get_notification_for_job(service_id, job_id, notification_id=None): except NoResultFound: return jsonify(result="error", message="Notification not found"), 404 else: - notifications = get_notifications(service_id, job_id) + notifications = notifications_dao.get_notifications_for_job(service_id, job_id) data, errors = notifications_status_schema.dump(notifications) return jsonify(data=data) @@ -116,13 +112,13 @@ def get_notification_for_job(service_id, job_id, notification_id=None): @job.route('//notification/', methods=['PUT']) def update_notification_for_job(service_id, job_id, notification_id): - notification = get_notification(service_id, job_id, notification_id) + notification = notifications_dao.get_notification_for_job(service_id, job_id, notification_id) update_dict, errors = notification_status_schema_load_json.load(request.get_json()) if errors: return jsonify(result="error", message=errors), 400 try: - save_notification(notification, update_dict=update_dict) + notifications_dao.save_notification(notification, update_dict=update_dict) except Exception as e: return jsonify(result="error", message=str(e)), 400 diff --git a/app/notifications/rest.py b/app/notifications/rest.py index 8bf50002e..6a9a888c2 100644 --- a/app/notifications/rest.py +++ b/app/notifications/rest.py @@ -4,16 +4,20 @@ from flask import ( Blueprint, jsonify, request, - current_app) -from itsdangerous import URLSafeSerializer + current_app +) +from itsdangerous import URLSafeSerializer from app import api_user from app.aws_sqs import add_notification_to_queue -from app.dao import (templates_dao) +from app.dao import (templates_dao, notifications_dao) from app.schemas import ( - email_notification_schema, sms_template_notification_schema) + email_notification_schema, + sms_template_notification_schema, + notification_status_schema +) from app.celery.tasks import send_sms - +from sqlalchemy.orm.exc import NoResultFound notifications = Blueprint('notifications', __name__) @@ -22,10 +26,13 @@ def create_notification_id(): return str(uuid.uuid4()) -@notifications.route('/', methods=['GET']) +@notifications.route('/', methods=['GET']) def get_notifications(notification_id): - # TODO return notification id details - return jsonify({'id': notification_id}), 200 + try: + notification = notifications_dao.get_notification(api_user['client'], notification_id) + return jsonify({'notification': notification_status_schema.dump(notification).data}), 200 + except NoResultFound: + return jsonify(result="error", message="not found"), 404 @notifications.route('/sms', methods=['POST']) diff --git a/app/schemas.py b/app/schemas.py index 050e35d48..b8fbbd757 100644 --- a/app/schemas.py +++ b/app/schemas.py @@ -141,7 +141,7 @@ class EmailNotificationSchema(NotificationSchema): class NotificationStatusSchema(BaseSchema): class Meta: - model = models.Notification + model = models.Noti∫~fication user_schema = UserSchema() diff --git a/tests/app/dao/test_notification_dao.py b/tests/app/dao/test_notification_dao.py index a28851520..384be5bff 100644 --- a/tests/app/dao/test_notification_dao.py +++ b/tests/app/dao/test_notification_dao.py @@ -3,7 +3,8 @@ from app.models import Notification from app.dao.notifications_dao import ( save_notification, get_notification, - get_notifications + get_notification_for_job, + get_notifications_for_job ) @@ -31,6 +32,13 @@ def test_save_notification(notify_db, notify_db_session, sample_template, sample assert 'sent' == notification_from_db.status +def test_get_notification(notify_db, notify_db_session, sample_notification): + notifcation_from_db = get_notification( + sample_notification.service.id, + sample_notification.id) + assert sample_notification == notifcation_from_db + + def test_save_notification_no_job_id(notify_db, notify_db_session, sample_template): assert Notification.query.count() == 0 @@ -54,9 +62,10 @@ def test_save_notification_no_job_id(notify_db, notify_db_session, sample_templa def test_get_notification_for_job(notify_db, notify_db_session, sample_notification): - notifcation_from_db = get_notification(sample_notification.service.id, - sample_notification.job_id, - sample_notification.id) + notifcation_from_db = get_notification_for_job( + sample_notification.service.id, + sample_notification.job_id, + sample_notification.id) assert sample_notification == notifcation_from_db @@ -70,7 +79,7 @@ def test_get_all_notifications_for_job(notify_db, notify_db_session, sample_job) template=sample_job.template, job=sample_job) - notifcations_from_db = get_notifications(sample_job.service.id, sample_job.id) + notifcations_from_db = get_notifications_for_job(sample_job.service.id, sample_job.id) assert len(notifcations_from_db) == 5 diff --git a/tests/app/notifications/test_rest.py b/tests/app/notifications/test_rest.py index 33cef53b0..dbe8be2b6 100644 --- a/tests/app/notifications/test_rest.py +++ b/tests/app/notifications/test_rest.py @@ -4,46 +4,49 @@ import uuid from tests import create_authorization_header from flask import url_for, json from app.models import Service -from tests.app.conftest import sample_service as create_sample_service -from tests.app.conftest import sample_template as create_sample_template def test_get_notifications( - notify_api, notify_db, notify_db_session, sample_api_key, mocker): + notify_api, notify_db, notify_db_session, sample_api_key, sample_notification): """ Tests GET endpoint '/' to retrieve entire service list. """ with notify_api.test_request_context(): with notify_api.test_client() as client: auth_header = create_authorization_header( - service_id=sample_api_key.service_id, - path=url_for('notifications.get_notifications', notification_id=123), + service_id=sample_notification.service_id, + path='/notifications/{}'.format(sample_notification.id), method='GET') response = client.get( - url_for('notifications.get_notifications', notification_id=123), + '/notifications/{}'.format(sample_notification.id), headers=[auth_header]) + notification = json.loads(response.get_data(as_text=True))['notification'] + assert notification['status'] == 'sent' + assert notification['template'] == sample_notification.template.id + assert notification['to'] == '+44709123456' + assert notification['service'] == str(sample_notification.service_id) assert response.status_code == 200 def test_get_notifications_empty_result( notify_api, notify_db, notify_db_session, sample_api_key, mocker): - """ - Tests GET endpoint '/' to retrieve entire service list. - """ with notify_api.test_request_context(): with notify_api.test_client() as client: + + id = uuid.uuid4() + auth_header = create_authorization_header( service_id=sample_api_key.service_id, - path=url_for('notifications.get_notifications', notification_id=123), + path=url_for('notifications.get_notifications', notification_id=id), method='GET') response = client.get( - url_for('notifications.get_notifications', notification_id=123), + url_for('notifications.get_notifications', notification_id=id), headers=[auth_header]) - assert response.status_code == 200 + assert response.status_code == 404 def test_create_sms_should_reject_if_no_phone_numbers( From e42da7dd54da8d047d171464579f4089c2e9c7c9 Mon Sep 17 00:00:00 2001 From: Martyn Inglis Date: Tue, 16 Feb 2016 14:06:56 +0000 Subject: [PATCH 05/17] Fixing up tests to validate the call to the celery tasks. - mocker used to test call or otherwise of the task - no new tests just a spring clean --- app/__init__.py | 4 +- app/celery/tasks.py | 4 +- app/notifications/rest.py | 9 +- app/schemas.py | 9 +- run_celery.py | 2 +- tests/app/conftest.py | 14 +- tests/app/notifications/test_rest.py | 223 +++++++++++++-------------- 7 files changed, 129 insertions(+), 136 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index f035df8ed..aa64ec5d6 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -17,7 +17,7 @@ from app.clients.sms.twilio import TwilioClient db = SQLAlchemy() ma = Marshmallow() notify_alpha_client = NotifyAPIClient() -celery = NotifyCelery() +notify_celery = NotifyCelery() twilio_client = TwilioClient() @@ -35,7 +35,7 @@ def create_app(config_name, config_overrides=None): ma.init_app(application) logging.init_app(application) twilio_client.init_app(application) - celery.init_app(application) + notify_celery.init_app(application) from app.service.rest import service as service_blueprint from app.user.rest import user as user_blueprint diff --git a/app/celery/tasks.py b/app/celery/tasks.py index e8d20d009..f17b844d7 100644 --- a/app/celery/tasks.py +++ b/app/celery/tasks.py @@ -1,12 +1,12 @@ from itsdangerous import URLSafeSerializer -from app import celery, twilio_client, db +from app import notify_celery, twilio_client, db from app.clients.sms.twilio import TwilioClientException from app.dao.templates_dao import get_model_templates from app.models import Notification from flask import current_app -@celery.task(name="send-sms", bind="True") +@notify_celery.task(name="send-sms", bind="True") def send_sms(service_id, notification_id, encrypted_notification, secret_key, salt): serializer = URLSafeSerializer(secret_key) diff --git a/app/notifications/rest.py b/app/notifications/rest.py index 6a9a888c2..51974437d 100644 --- a/app/notifications/rest.py +++ b/app/notifications/rest.py @@ -39,12 +39,15 @@ def get_notifications(notification_id): def create_sms_notification(): serializer = URLSafeSerializer(current_app.config.get('SECRET_KEY')) - resp_json = request.get_json() - - notification, errors = sms_template_notification_schema.load(resp_json) + notification, errors = sms_template_notification_schema.load(request.get_json()) if errors: return jsonify(result="error", message=errors), 400 + try: + templates_dao.get_model_templates(template_id=notification['template'], service_id=api_user['client']) + except NoResultFound: + return jsonify(result="error", message={'template': ['Template not found']}), 400 + notification_id = create_notification_id() encrypted_notification = serializer.dumps(notification, current_app.config.get('DANGEROUS_SALT')) diff --git a/app/schemas.py b/app/schemas.py index b8fbbd757..9810cad6d 100644 --- a/app/schemas.py +++ b/app/schemas.py @@ -93,13 +93,6 @@ class SmsTemplateNotificationSchema(SmsNotificationSchema): template = fields.Int(required=True) job = fields.String() - @validates('template') - def validate_template(self, value): - if not models.Template.query.filter_by(id=value).first(): - # TODO is this message consistent with what marshmallow - # would normally produce. - raise ValidationError('Template not found') - @validates_schema def validate_schema(self, data): """ @@ -141,7 +134,7 @@ class EmailNotificationSchema(NotificationSchema): class NotificationStatusSchema(BaseSchema): class Meta: - model = models.Noti∫~fication + model = models.Notification user_schema = UserSchema() diff --git a/run_celery.py b/run_celery.py index 269142d5b..46217adae 100644 --- a/run_celery.py +++ b/run_celery.py @@ -1,6 +1,6 @@ #!/usr/bin/env python import os -from app import celery, create_app +from app import notify_celery, create_app application = create_app(os.getenv('NOTIFY_API_ENVIRONMENT') or 'development') application.app_context().push() diff --git a/tests/app/conftest.py b/tests/app/conftest.py index 26d5179a9..906ce664d 100644 --- a/tests/app/conftest.py +++ b/tests/app/conftest.py @@ -1,7 +1,6 @@ import pytest -from flask import jsonify -from app.models import (User, Service, Template, ApiKey, Job, VerifyCode, Notification) +from app.models import (User, Service, Template, ApiKey, Job, Notification) from app.dao.users_dao import (save_model_user, create_user_code, create_secret_code) from app.dao.services_dao import save_model_service from app.dao.templates_dao import save_model_template @@ -10,6 +9,17 @@ from app.dao.jobs_dao import save_job from app.dao.notifications_dao import save_notification import uuid +@pytest.fixture(scope='function') +def service_factory(notify_db, notify_db_session): + class ServiceFactory(object): + def get(self, service_name): + user = sample_user(notify_db, notify_db_session) + service = sample_service(notify_db, notify_db_session, service_name, user) + sample_template(notify_db, notify_db_session, service=service) + return service + + return ServiceFactory() + @pytest.fixture(scope='function') def sample_user(notify_db, diff --git a/tests/app/notifications/test_rest.py b/tests/app/notifications/test_rest.py index dbe8be2b6..63f175b43 100644 --- a/tests/app/notifications/test_rest.py +++ b/tests/app/notifications/test_rest.py @@ -1,16 +1,14 @@ -import moto import uuid +import app.celery.tasks from tests import create_authorization_header -from flask import url_for, json +from flask import json from app.models import Service +from app.dao.templates_dao import get_model_templates +from mock import ANY -def test_get_notifications( - notify_api, notify_db, notify_db_session, sample_api_key, sample_notification): - """ - Tests GET endpoint '/' to retrieve entire service list. - """ +def test_get_notification_by_id(notify_api, sample_notification): with notify_api.test_request_context(): with notify_api.test_client() as client: auth_header = create_authorization_header( @@ -30,217 +28,208 @@ def test_get_notifications( assert response.status_code == 200 -def test_get_notifications_empty_result( - notify_api, notify_db, notify_db_session, sample_api_key, mocker): +def test_get_notifications_empty_result(notify_api, sample_api_key): with notify_api.test_request_context(): with notify_api.test_client() as client: - id = uuid.uuid4() - + missing_notification_id = uuid.uuid4() auth_header = create_authorization_header( service_id=sample_api_key.service_id, - path=url_for('notifications.get_notifications', notification_id=id), + path='/notifications/{}'.format(missing_notification_id), method='GET') response = client.get( - url_for('notifications.get_notifications', notification_id=id), + path='/notifications/{}'.format(missing_notification_id), headers=[auth_header]) + notification = json.loads(response.get_data(as_text=True)) + assert notification['result'] == "error" + assert notification['message'] == "not found" assert response.status_code == 404 -def test_create_sms_should_reject_if_no_phone_numbers( - notify_api, notify_db, notify_db_session, sample_api_key, mocker): - """ - Tests GET endpoint '/' to retrieve entire service list. - """ +def test_create_sms_should_reject_if_missing_required_fields(notify_api, sample_api_key, mocker): with notify_api.test_request_context(): with notify_api.test_client() as client: - data = { - 'template': "my message" - } + mocker.patch('app.celery.tasks.send_sms.apply_async') + + data = {} auth_header = create_authorization_header( service_id=sample_api_key.service_id, request_body=json.dumps(data), - path=url_for('notifications.create_sms_notification'), + path='/notifications/sms', method='POST') response = client.post( - url_for('notifications.create_sms_notification'), + path='/notifications/sms', data=json.dumps(data), headers=[('Content-Type', 'application/json'), auth_header]) json_resp = json.loads(response.get_data(as_text=True)) - assert response.status_code == 400 + app.celery.tasks.send_sms.apply_async.assert_not_called() assert json_resp['result'] == 'error' assert 'Missing data for required field.' in json_resp['message']['to'][0] + assert 'Missing data for required field.' in json_resp['message']['template'][0] + assert response.status_code == 400 -def test_should_reject_bad_phone_numbers( - notify_api, notify_db, notify_db_session, mocker): - """ - Tests GET endpoint '/' to retrieve entire service list. - """ +def test_should_reject_bad_phone_numbers(notify_api, sample_template, mocker): with notify_api.test_request_context(): with notify_api.test_client() as client: + mocker.patch('app.celery.tasks.send_sms.apply_async') + data = { 'to': 'invalid', - 'template': "my message" + 'template': sample_template.id } auth_header = create_authorization_header( request_body=json.dumps(data), - path=url_for('notifications.create_sms_notification'), + path='/notifications/sms', method='POST') response = client.post( - url_for('notifications.create_sms_notification'), + path='/notifications/sms', data=json.dumps(data), headers=[('Content-Type', 'application/json'), auth_header]) json_resp = json.loads(response.get_data(as_text=True)) - assert response.status_code == 400 + app.celery.tasks.send_sms.apply_async.assert_not_called() + assert json_resp['result'] == 'error' + assert len(json_resp['message'].keys()) == 1 assert 'Invalid phone number, must be of format +441234123123' in json_resp['message']['to'] + assert response.status_code == 400 -def test_send_notification_restrict_mobile(notify_api, - notify_db, - notify_db_session, - sample_api_key, - sample_template, - sample_user, - mocker): - """ - Test POST endpoint '/sms' with service notification with mobile number - not in restricted list. - """ +def test_send_notification_invalid_template_id(notify_api, sample_template, mocker): with notify_api.test_request_context(): with notify_api.test_client() as client: + mocker.patch('app.celery.tasks.send_sms.apply_async') + + data = { + 'to': '+441234123123', + 'template': 9999 + } + auth_header = create_authorization_header( + service_id=sample_template.service.id, + request_body=json.dumps(data), + path='/notifications/sms', + method='POST') + + response = client.post( + path='/notifications/sms', + data=json.dumps(data), + headers=[('Content-Type', 'application/json'), auth_header]) + + json_resp = json.loads(response.get_data(as_text=True)) + app.celery.tasks.send_sms.apply_async.assert_not_called() + + assert response.status_code == 400 + assert len(json_resp['message'].keys()) == 1 + assert 'Template not found' in json_resp['message']['template'] + + +def test_prevents_sending_to_any_mobile_on_restricted_service(notify_api, sample_template, mocker): + with notify_api.test_request_context(): + with notify_api.test_client() as client: + mocker.patch('app.celery.tasks.send_sms.apply_async') + Service.query.filter_by( - id=sample_template.service.id).update({'restricted': True}) + id=sample_template.service.id + ).update( + {'restricted': True} + ) invalid_mob = '+449999999999' data = { 'to': invalid_mob, 'template': sample_template.id } - assert invalid_mob != sample_user.mobile_number + auth_header = create_authorization_header( service_id=sample_template.service.id, request_body=json.dumps(data), - path=url_for('notifications.create_sms_notification'), + path='/notifications/sms', method='POST') response = client.post( - url_for('notifications.create_sms_notification'), + path='/notifications/sms', data=json.dumps(data), headers=[('Content-Type', 'application/json'), auth_header]) json_resp = json.loads(response.get_data(as_text=True)) + app.celery.tasks.send_sms.apply_async.assert_not_called() + assert response.status_code == 400 assert 'Invalid phone number for restricted service' in json_resp['message']['restricted'] -def test_send_notification_invalid_template_id(notify_api, - notify_db, - notify_db_session, - sample_api_key, - sample_template, - sample_user, - mocker): - """ - Tests POST endpoint '/sms' with notifications-admin notification with invalid template id - """ +def test_should_not_allow_template_from_another_service(notify_api, service_factory, mocker): with notify_api.test_request_context(): with notify_api.test_client() as client: + mocker.patch('app.celery.tasks.send_sms.apply_async') - Service.query.filter_by( - id=sample_template.service.id).update({'restricted': True}) - invalid_mob = '+449999999999' - data = { - 'to': invalid_mob, - 'template': 9999 - } - assert invalid_mob != sample_user.mobile_number - auth_header = create_authorization_header( - service_id=sample_template.service.id, - request_body=json.dumps(data), - path=url_for('notifications.create_sms_notification'), - method='POST') + service_1 = service_factory.get('service 1') + service_2 = service_factory.get('service 2') - response = client.post( - url_for('notifications.create_sms_notification'), - data=json.dumps(data), - headers=[('Content-Type', 'application/json'), auth_header]) + service_2_templates = get_model_templates(service_id=service_2.id) - json_resp = json.loads(response.get_data(as_text=True)) - assert response.status_code == 400 - assert 'Template not found' in json_resp['message']['template'] - - -@moto.mock_sqs -def test_should_not_allow_template_from_other_service(notify_api, - notify_db, - notify_db_session, - sample_template, - sample_admin_service_id, - mocker): - """ - Tests POST endpoint '/sms' with notifications. - """ - with notify_api.test_request_context(): - with notify_api.test_client() as client: data = { 'to': '+441234123123', - 'template': sample_template.id + 'template': service_2_templates[0].id } + auth_header = create_authorization_header( - service_id=sample_admin_service_id, + service_id=service_1.id, request_body=json.dumps(data), - path=url_for('notifications.create_sms_notification'), + path='/notifications/sms', method='POST') response = client.post( - url_for('notifications.create_sms_notification'), + path='/notifications/sms', data=json.dumps(data), headers=[('Content-Type', 'application/json'), auth_header]) json_resp = json.loads(response.get_data(as_text=True)) + app.celery.tasks.send_sms.apply_async.assert_not_called() + assert response.status_code == 400 assert 'Invalid template' in json_resp['message']['restricted'] -@moto.mock_sqs -def test_should_allow_valid_message(notify_api, - notify_db, - notify_db_session, - sqs_client_conn, - sample_user, - sample_template, - mocker): - """ - Tests POST endpoint '/sms' with notifications-admin notification. - """ +def test_should_allow_valid_sms_notification(notify_api, sample_template, mocker): with notify_api.test_request_context(): with notify_api.test_client() as client: + mocker.patch('app.celery.tasks.send_sms.apply_async') + data = { 'to': '+441234123123', 'template': sample_template.id } + auth_header = create_authorization_header( request_body=json.dumps(data), - path=url_for('notifications.create_sms_notification'), - method='POST') + path='/notifications/sms', + method='POST', + service_id=sample_template.service_id + ) response = client.post( - url_for('notifications.create_sms_notification'), + path='/notifications/sms', data=json.dumps(data), headers=[('Content-Type', 'application/json'), auth_header]) + notification_id = json.loads(response.data)['notification_id'] + app.celery.tasks.send_sms.apply_async.assert_called_once_with( + (str(sample_template.service_id), + notification_id, + ANY, + notify_api.config['SECRET_KEY'], + notify_api.config['DANGEROUS_SALT']) + ) assert response.status_code == 201 - assert json.loads(response.data)['notification_id'] is not None + assert notification_id -@moto.mock_sqs def test_send_email_valid_data(notify_api, notify_db, notify_db_session, @@ -262,11 +251,11 @@ def test_send_email_valid_data(notify_api, } auth_header = create_authorization_header( request_body=json.dumps(data), - path=url_for('notifications.create_email_notification'), + path='/notifications/email', method='POST') response = client.post( - url_for('notifications.create_email_notification'), + path='/notifications/email', data=json.dumps(data), headers=[('Content-Type', 'application/json'), auth_header]) @@ -274,7 +263,6 @@ def test_send_email_valid_data(notify_api, assert json.loads(response.data)['notification_id'] is not None -@moto.mock_sqs def test_valid_message_with_service_id(notify_api, notify_db, notify_db_session, @@ -286,7 +274,7 @@ def test_valid_message_with_service_id(notify_api, with notify_api.test_client() as client: job_id = uuid.uuid4() service_id = sample_template.service.id - url = url_for('notifications.create_sms_for_service', service_id=service_id) + url = '/notifications/sms/service/{}'.format(service_id) data = { 'to': '+441234123123', 'template': sample_template.id, @@ -306,7 +294,6 @@ def test_valid_message_with_service_id(notify_api, assert json.loads(response.data)['notification_id'] is not None -@moto.mock_sqs def test_message_with_incorrect_service_id_should_fail(notify_api, notify_db, notify_db_session, @@ -319,7 +306,7 @@ def test_message_with_incorrect_service_id_should_fail(notify_api, job_id = uuid.uuid4() invalid_service_id = uuid.uuid4() - url = url_for('notifications.create_sms_for_service', service_id=invalid_service_id) + url = '/notifications/sms/service/{}'.format(invalid_service_id) data = { 'to': '+441234123123', From 18d63e241ba46220603e4e9889c231f82102c95f Mon Sep 17 00:00:00 2001 From: Martyn Inglis Date: Tue, 16 Feb 2016 15:28:30 +0000 Subject: [PATCH 06/17] Extracted serialiser for encryption into a flask module - allows mocking easier - shared across methods - not built everytime --- app/__init__.py | 4 +++- app/celery/tasks.py | 8 +++----- app/encryption.py | 15 +++++++++++++++ app/notifications/rest.py | 9 ++------- tests/app/notifications/test_rest.py | 5 ++--- tests/app/test_encryption.py | 20 ++++++++++++++++++++ tests/conftest.py | 1 + 7 files changed, 46 insertions(+), 16 deletions(-) create mode 100644 tests/app/test_encryption.py diff --git a/app/__init__.py b/app/__init__.py index aa64ec5d6..524d2f14f 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -13,13 +13,14 @@ from utils import logging from notify_client import NotifyAPIClient from app.celery.celery import NotifyCelery from app.clients.sms.twilio import TwilioClient +from app.encryption import Encryption db = SQLAlchemy() ma = Marshmallow() notify_alpha_client = NotifyAPIClient() notify_celery = NotifyCelery() twilio_client = TwilioClient() - +encryption = Encryption() api_user = LocalProxy(lambda: _request_ctx_stack.top.api_user) @@ -36,6 +37,7 @@ def create_app(config_name, config_overrides=None): logging.init_app(application) twilio_client.init_app(application) notify_celery.init_app(application) + encryption.init_app(application) from app.service.rest import service as service_blueprint from app.user.rest import user as user_blueprint diff --git a/app/celery/tasks.py b/app/celery/tasks.py index f17b844d7..7f0507bef 100644 --- a/app/celery/tasks.py +++ b/app/celery/tasks.py @@ -1,5 +1,5 @@ from itsdangerous import URLSafeSerializer -from app import notify_celery, twilio_client, db +from app import notify_celery, twilio_client, db, encryption from app.clients.sms.twilio import TwilioClientException from app.dao.templates_dao import get_model_templates from app.models import Notification @@ -7,10 +7,8 @@ from flask import current_app @notify_celery.task(name="send-sms", bind="True") -def send_sms(service_id, notification_id, encrypted_notification, secret_key, salt): - serializer = URLSafeSerializer(secret_key) - - notification = serializer.loads(encrypted_notification, salt=salt) +def send_sms(service_id, notification_id, encrypted_notification): + notification = encryption.decrypt(encrypted_notification) template = get_model_templates(notification['template']) status = 'sent' diff --git a/app/encryption.py b/app/encryption.py index 51caaab72..920a5f9fb 100644 --- a/app/encryption.py +++ b/app/encryption.py @@ -1,5 +1,20 @@ from flask.ext.bcrypt import generate_password_hash, check_password_hash +from itsdangerous import URLSafeSerializer + + +class Encryption: + + def init_app(self, app): + self.serializer = URLSafeSerializer(app.config.get('SECRET_KEY')) + self.salt = app.config.get('DANGEROUS_SALT') + + def encrypt(self, thing_to_encrypt): + return self.serializer.dumps(thing_to_encrypt, self.salt) + + def decrypt(self, thing_to_decrypt): + return self.serializer.loads(thing_to_decrypt, salt=self.salt) + def hashpw(password): return generate_password_hash(password.encode('UTF-8'), 10) diff --git a/app/notifications/rest.py b/app/notifications/rest.py index 51974437d..3f1726dda 100644 --- a/app/notifications/rest.py +++ b/app/notifications/rest.py @@ -8,7 +8,7 @@ from flask import ( ) from itsdangerous import URLSafeSerializer -from app import api_user +from app import api_user, encryption from app.aws_sqs import add_notification_to_queue from app.dao import (templates_dao, notifications_dao) from app.schemas import ( @@ -37,8 +37,6 @@ def get_notifications(notification_id): @notifications.route('/sms', methods=['POST']) def create_sms_notification(): - serializer = URLSafeSerializer(current_app.config.get('SECRET_KEY')) - notification, errors = sms_template_notification_schema.load(request.get_json()) if errors: return jsonify(result="error", message=errors), 400 @@ -49,14 +47,11 @@ def create_sms_notification(): return jsonify(result="error", message={'template': ['Template not found']}), 400 notification_id = create_notification_id() - encrypted_notification = serializer.dumps(notification, current_app.config.get('DANGEROUS_SALT')) send_sms.apply_async(( api_user['client'], notification_id, - encrypted_notification, - current_app.config.get('SECRET_KEY'), - current_app.config.get('DANGEROUS_SALT'))) + encryption.encrypt(notification))) return jsonify({'notification_id': notification_id}), 201 diff --git a/tests/app/notifications/test_rest.py b/tests/app/notifications/test_rest.py index 63f175b43..ae931c930 100644 --- a/tests/app/notifications/test_rest.py +++ b/tests/app/notifications/test_rest.py @@ -200,6 +200,7 @@ def test_should_allow_valid_sms_notification(notify_api, sample_template, mocker with notify_api.test_request_context(): with notify_api.test_client() as client: mocker.patch('app.celery.tasks.send_sms.apply_async') + mocker.patch('app.encryption.encrypt', return_value="something_encrypted") data = { 'to': '+441234123123', @@ -222,9 +223,7 @@ def test_should_allow_valid_sms_notification(notify_api, sample_template, mocker app.celery.tasks.send_sms.apply_async.assert_called_once_with( (str(sample_template.service_id), notification_id, - ANY, - notify_api.config['SECRET_KEY'], - notify_api.config['DANGEROUS_SALT']) + "something_encrypted") ) assert response.status_code == 201 assert notification_id diff --git a/tests/app/test_encryption.py b/tests/app/test_encryption.py new file mode 100644 index 000000000..d3566b202 --- /dev/null +++ b/tests/app/test_encryption.py @@ -0,0 +1,20 @@ +from app.encryption import Encryption + +encryption = Encryption() + + +def test_should_encrypt_content(notify_api): + encryption.init_app(notify_api) + assert encryption.encrypt("this") != "this" + + +def test_should_decrypt_content(notify_api): + encryption.init_app(notify_api) + encrypted = encryption.encrypt("this") + assert encryption.decrypt(encrypted) == "this" + + +def test_should_encrypt_json(notify_api): + encryption.init_app(notify_api) + encrypted = encryption.encrypt({"this": "that"}) + assert encryption.decrypt(encrypted) == {"this": "that"} diff --git a/tests/conftest.py b/tests/conftest.py index 37ee9bbd3..f7404b8ed 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -63,6 +63,7 @@ def notify_db_session(request): def notify_config(notify_api): notify_api.config['NOTIFY_API_ENVIRONMENT'] = 'test' notify_api.config.from_object(configs['test']) + return notify_api.config @pytest.fixture(scope='function') From 0933e5c64773ca305eadd0d46ec8f805a7489476 Mon Sep 17 00:00:00 2001 From: Martyn Inglis Date: Tue, 16 Feb 2016 17:17:02 +0000 Subject: [PATCH 07/17] Building tests for the tasks class --- app/celery/tasks.py | 9 ++-- app/encryption.py | 3 +- app/notifications/rest.py | 4 +- tests/app/celery/test_tasks.py | 74 ++++++++++++++++++++++++++++ tests/app/notifications/test_rest.py | 1 - 5 files changed, 81 insertions(+), 10 deletions(-) create mode 100644 tests/app/celery/test_tasks.py diff --git a/app/celery/tasks.py b/app/celery/tasks.py index 7f0507bef..b446092c3 100644 --- a/app/celery/tasks.py +++ b/app/celery/tasks.py @@ -1,12 +1,12 @@ -from itsdangerous import URLSafeSerializer from app import notify_celery, twilio_client, db, encryption from app.clients.sms.twilio import TwilioClientException from app.dao.templates_dao import get_model_templates +from app.dao.notifications_dao import save_notification from app.models import Notification from flask import current_app -@notify_celery.task(name="send-sms", bind="True") +@notify_celery.task(name="send-sms") def send_sms(service_id, notification_id, encrypted_notification): notification = encryption.decrypt(encrypted_notification) template = get_model_templates(notification['template']) @@ -14,7 +14,7 @@ def send_sms(service_id, notification_id, encrypted_notification): status = 'sent' try: - twilio_client.send_sms(notification, template.content) + twilio_client.send_sms(notification['to'], template.content) except TwilioClientException as e: current_app.logger.info(e) status = 'failed' @@ -27,5 +27,4 @@ def send_sms(service_id, notification_id, encrypted_notification): status=status ) - db.session.add(notification_db_object) - db.session.commit() + save_notification(notification_db_object) diff --git a/app/encryption.py b/app/encryption.py index 920a5f9fb..ffe055dc9 100644 --- a/app/encryption.py +++ b/app/encryption.py @@ -10,7 +10,8 @@ class Encryption: self.salt = app.config.get('DANGEROUS_SALT') def encrypt(self, thing_to_encrypt): - return self.serializer.dumps(thing_to_encrypt, self.salt) + return self.serializer.dumps(thing_to_encrypt, salt= + self.salt) def decrypt(self, thing_to_decrypt): return self.serializer.loads(thing_to_decrypt, salt=self.salt) diff --git a/app/notifications/rest.py b/app/notifications/rest.py index 3f1726dda..ecf34bcb6 100644 --- a/app/notifications/rest.py +++ b/app/notifications/rest.py @@ -3,11 +3,9 @@ import uuid from flask import ( Blueprint, jsonify, - request, - current_app + request ) -from itsdangerous import URLSafeSerializer from app import api_user, encryption from app.aws_sqs import add_notification_to_queue from app.dao import (templates_dao, notifications_dao) diff --git a/tests/app/celery/test_tasks.py b/tests/app/celery/test_tasks.py new file mode 100644 index 000000000..dfb79c4d8 --- /dev/null +++ b/tests/app/celery/test_tasks.py @@ -0,0 +1,74 @@ +import uuid +from app.celery.tasks import send_sms +from app import twilio_client +from app.clients.sms.twilio import TwilioClientException +from app.dao import notifications_dao + + +def test_should_send_template_to_correct_sms_provider_and_persist(sample_template, mocker): + notification = { + "template": sample_template.id, + "to": "+441234123123" + } + mocker.patch('app.encryption.decrypt', return_value=notification) + mocker.patch('app.twilio_client.send_sms') + + notification_id = uuid.uuid4() + + send_sms( + sample_template.service_id, + notification_id, + "encrypted-in-reality") + + twilio_client.send_sms.assert_called_once_with("+441234123123", sample_template.content) + persisted_notification = notifications_dao.get_notification(sample_template.service_id, notification_id) + assert persisted_notification.id == notification_id + assert persisted_notification.to == '+441234123123' + assert persisted_notification.template_id == sample_template.id + assert persisted_notification.status == 'sent' + + +def test_should_persist_notification_as_failed_if_sms_client_fails(sample_template, mocker): + notification = { + "template": sample_template.id, + "to": "+441234123123" + } + mocker.patch('app.encryption.decrypt', return_value=notification) + mocker.patch('app.twilio_client.send_sms', side_effect=TwilioClientException()) + + notification_id = uuid.uuid4() + + send_sms( + sample_template.service_id, + notification_id, + "encrypted-in-reality") + + twilio_client.send_sms.assert_called_once_with("+441234123123", sample_template.content) + persisted_notification = notifications_dao.get_notification(sample_template.service_id, notification_id) + assert persisted_notification.id == notification_id + assert persisted_notification.to == '+441234123123' + assert persisted_notification.template_id == sample_template.id + assert persisted_notification.status == 'failed' + + +def test_should_not_send_sms_if_db_peristance_failed(sample_template, mocker): + notification = { + "template": sample_template.id, + "to": "+441234123123" + } + mocker.patch('app.encryption.decrypt', return_value=notification) + mocker.patch('app.twilio_client.send_sms', side_effect=TwilioClientException()) + + notification_id = uuid.uuid4() + + send_sms( + sample_template.service_id, + notification_id, + "encrypted-in-reality") + + twilio_client.send_sms.assert_called_once_with("+441234123123", sample_template.content) + persisted_notification = notifications_dao.get_notification(sample_template.service_id, notification_id) + assert persisted_notification.id == notification_id + assert persisted_notification.to == '+441234123123' + assert persisted_notification.template_id == sample_template.id + assert persisted_notification.status == 'failed' diff --git a/tests/app/notifications/test_rest.py b/tests/app/notifications/test_rest.py index ae931c930..46639263e 100644 --- a/tests/app/notifications/test_rest.py +++ b/tests/app/notifications/test_rest.py @@ -5,7 +5,6 @@ from tests import create_authorization_header from flask import json from app.models import Service from app.dao.templates_dao import get_model_templates -from mock import ANY def test_get_notification_by_id(notify_api, sample_notification): From a2341be0e2c200c1e8dc541b0608876f6b09739c Mon Sep 17 00:00:00 2001 From: Martyn Inglis Date: Tue, 16 Feb 2016 17:42:04 +0000 Subject: [PATCH 08/17] Don't send the SMS if we have failed to save in the database --- app/celery/tasks.py | 30 ++++++++++++++++-------------- app/encryption.py | 4 +--- tests/app/celery/test_tasks.py | 14 ++++++++------ tests/app/conftest.py | 1 + 4 files changed, 26 insertions(+), 23 deletions(-) diff --git a/app/celery/tasks.py b/app/celery/tasks.py index b446092c3..8806ec6ed 100644 --- a/app/celery/tasks.py +++ b/app/celery/tasks.py @@ -4,6 +4,7 @@ from app.dao.templates_dao import get_model_templates from app.dao.notifications_dao import save_notification from app.models import Notification from flask import current_app +from sqlalchemy.exc import SQLAlchemyError @notify_celery.task(name="send-sms") @@ -11,20 +12,21 @@ def send_sms(service_id, notification_id, encrypted_notification): notification = encryption.decrypt(encrypted_notification) template = get_model_templates(notification['template']) - status = 'sent' - try: - twilio_client.send_sms(notification['to'], template.content) - except TwilioClientException as e: - current_app.logger.info(e) - status = 'failed' + notification_db_object = Notification( + id=notification_id, + template_id=notification['template'], + to=notification['to'], + service_id=service_id, + status='sent' + ) + save_notification(notification_db_object) - notification_db_object = Notification( - id=notification_id, - template_id=notification['template'], - to=notification['to'], - service_id=service_id, - status=status - ) + try: + twilio_client.send_sms(notification['to'], template.content) + except TwilioClientException as e: + current_app.logger.debug(e) + save_notification(notification_db_object, {"status": "failed"}) - save_notification(notification_db_object) + except SQLAlchemyError as e: + current_app.logger.debug(e) diff --git a/app/encryption.py b/app/encryption.py index ffe055dc9..2b4755803 100644 --- a/app/encryption.py +++ b/app/encryption.py @@ -4,14 +4,12 @@ from itsdangerous import URLSafeSerializer class Encryption: - def init_app(self, app): self.serializer = URLSafeSerializer(app.config.get('SECRET_KEY')) self.salt = app.config.get('DANGEROUS_SALT') def encrypt(self, thing_to_encrypt): - return self.serializer.dumps(thing_to_encrypt, salt= - self.salt) + return self.serializer.dumps(thing_to_encrypt, salt=self.salt) def decrypt(self, thing_to_decrypt): return self.serializer.loads(thing_to_decrypt, salt=self.salt) diff --git a/tests/app/celery/test_tasks.py b/tests/app/celery/test_tasks.py index dfb79c4d8..c53ae2469 100644 --- a/tests/app/celery/test_tasks.py +++ b/tests/app/celery/test_tasks.py @@ -1,8 +1,11 @@ import uuid +import pytest from app.celery.tasks import send_sms from app import twilio_client from app.clients.sms.twilio import TwilioClientException from app.dao import notifications_dao +from sqlalchemy.exc import SQLAlchemyError +from sqlalchemy.orm.exc import NoResultFound def test_should_send_template_to_correct_sms_provider_and_persist(sample_template, mocker): @@ -58,6 +61,7 @@ def test_should_not_send_sms_if_db_peristance_failed(sample_template, mocker): } mocker.patch('app.encryption.decrypt', return_value=notification) mocker.patch('app.twilio_client.send_sms', side_effect=TwilioClientException()) + mocker.patch('app.db.session.add', side_effect=SQLAlchemyError()) notification_id = uuid.uuid4() @@ -66,9 +70,7 @@ def test_should_not_send_sms_if_db_peristance_failed(sample_template, mocker): notification_id, "encrypted-in-reality") - twilio_client.send_sms.assert_called_once_with("+441234123123", sample_template.content) - persisted_notification = notifications_dao.get_notification(sample_template.service_id, notification_id) - assert persisted_notification.id == notification_id - assert persisted_notification.to == '+441234123123' - assert persisted_notification.template_id == sample_template.id - assert persisted_notification.status == 'failed' + twilio_client.send_sms.assert_not_called() + with pytest.raises(NoResultFound) as e: + notifications_dao.get_notification(sample_template.service_id, notification_id) + assert 'No row was found for one' in str(e.value) diff --git a/tests/app/conftest.py b/tests/app/conftest.py index 906ce664d..c3ba5c07b 100644 --- a/tests/app/conftest.py +++ b/tests/app/conftest.py @@ -9,6 +9,7 @@ from app.dao.jobs_dao import save_job from app.dao.notifications_dao import save_notification import uuid + @pytest.fixture(scope='function') def service_factory(notify_db, notify_db_session): class ServiceFactory(object): From 1311af59201b27bcc7eb960a8342672bd68a1c10 Mon Sep 17 00:00:00 2001 From: Martyn Inglis Date: Wed, 17 Feb 2016 09:12:13 +0000 Subject: [PATCH 09/17] Twilio client takes a 'to' number not a notification --- app/clients/sms/twilio.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/clients/sms/twilio.py b/app/clients/sms/twilio.py index 52b0375ee..eb5613878 100644 --- a/app/clients/sms/twilio.py +++ b/app/clients/sms/twilio.py @@ -23,11 +23,11 @@ class TwilioClient(SmsClient): config.config.get('TWILIO_AUTH_TOKEN')) self.from_number = config.config.get('TWILIO_NUMBER') - def send_sms(self, notification, content): + def send_sms(self, to, content): try: response = self.client.messages.create( body=content, - to=notification['to'], + to=to, from_=self.from_number ) return response.sid From 73d0e67e68c40be7ad2d8dd5fee8fdec8c470dae Mon Sep 17 00:00:00 2001 From: Martyn Inglis Date: Wed, 17 Feb 2016 09:22:06 +0000 Subject: [PATCH 10/17] Removed alpha api client --- app/__init__.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index dc9020dca..6b85f33ab 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -1,22 +1,17 @@ import os -import re -import ast from flask import request, url_for -from flask._compat import string_types from flask import Flask, _request_ctx_stack from flask.ext.sqlalchemy import SQLAlchemy from flask_marshmallow import Marshmallow from werkzeug.local import LocalProxy from utils import logging -from notify_client import NotifyAPIClient from app.celery.celery import NotifyCelery from app.clients.sms.twilio import TwilioClient from app.encryption import Encryption db = SQLAlchemy() ma = Marshmallow() -notify_alpha_client = NotifyAPIClient() notify_celery = NotifyCelery() twilio_client = TwilioClient() encryption = Encryption() From ec8955503fc8b28751eb2ee9633dff79ab3e981d Mon Sep 17 00:00:00 2001 From: Martyn Inglis Date: Wed, 17 Feb 2016 09:34:57 +0000 Subject: [PATCH 11/17] Twilio params into setup file for tests --- environment_test.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/environment_test.sh b/environment_test.sh index 094e3ddc1..2be312597 100644 --- a/environment_test.sh +++ b/environment_test.sh @@ -10,3 +10,6 @@ export NOTIFICATION_QUEUE_PREFIX='notification_development-test' export SECRET_KEY='secret-key' export SQLALCHEMY_DATABASE_URI='postgresql://localhost/test_notification_api' export VERIFY_CODE_FROM_EMAIL_ADDRESS='no-reply@notify.works' +export TWILIO_ACCOUNT_SID="test" +export TWILIO_AUTH_TOKEN="test" +export TWILIO_NUMBER="test" \ No newline at end of file From 85a8e6d2b4c055be0870ca14b8f77666261b61c7 Mon Sep 17 00:00:00 2001 From: Martyn Inglis Date: Wed, 17 Feb 2016 09:47:34 +0000 Subject: [PATCH 12/17] Added moto back for methods not yet celery'd --- tests/app/notifications/test_rest.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/app/notifications/test_rest.py b/tests/app/notifications/test_rest.py index 46639263e..326b1d72d 100644 --- a/tests/app/notifications/test_rest.py +++ b/tests/app/notifications/test_rest.py @@ -1,6 +1,6 @@ import uuid import app.celery.tasks - +import moto from tests import create_authorization_header from flask import json from app.models import Service @@ -228,6 +228,7 @@ def test_should_allow_valid_sms_notification(notify_api, sample_template, mocker assert notification_id +@moto.mock_sqs def test_send_email_valid_data(notify_api, notify_db, notify_db_session, @@ -261,6 +262,7 @@ def test_send_email_valid_data(notify_api, assert json.loads(response.data)['notification_id'] is not None +@moto.mock_sqs def test_valid_message_with_service_id(notify_api, notify_db, notify_db_session, @@ -292,6 +294,7 @@ def test_valid_message_with_service_id(notify_api, assert json.loads(response.data)['notification_id'] is not None +@moto.mock_sqs def test_message_with_incorrect_service_id_should_fail(notify_api, notify_db, notify_db_session, From e0e47b40fc196615b8698fde0290caab7766e949 Mon Sep 17 00:00:00 2001 From: Martyn Inglis Date: Wed, 17 Feb 2016 10:22:25 +0000 Subject: [PATCH 13/17] Setup celery config --- app/notifications/rest.py | 3 ++- config.py | 3 +-- run_celery.py | 4 +--- scripts/run_celery.sh | 6 ++++++ tests/app/notifications/test_rest.py | 4 ++-- 5 files changed, 12 insertions(+), 8 deletions(-) create mode 100755 scripts/run_celery.sh diff --git a/app/notifications/rest.py b/app/notifications/rest.py index ecf34bcb6..fdc3bc747 100644 --- a/app/notifications/rest.py +++ b/app/notifications/rest.py @@ -49,7 +49,8 @@ def create_sms_notification(): send_sms.apply_async(( api_user['client'], notification_id, - encryption.encrypt(notification))) + encryption.encrypt(notification)), + queue='sms') return jsonify({'notification_id': notification_id}), 201 diff --git a/config.py b/config.py index 55459c79d..a399e2883 100644 --- a/config.py +++ b/config.py @@ -1,5 +1,4 @@ import os -from datetime import timedelta class Config(object): @@ -26,7 +25,7 @@ class Config(object): 'region': 'eu-west-1', 'polling_interval': 10, # 1 second 'visibility_timeout': 3600, # 1 hour - 'queue_name_prefix': 'NOTIFY-CELERY-TEST-' + 'queue_name_prefix': 'preview' } CELERY_ENABLE_UTC = True, CELERY_TIMEZONE = 'Europe/London' diff --git a/run_celery.py b/run_celery.py index 46217adae..430577774 100644 --- a/run_celery.py +++ b/run_celery.py @@ -1,6 +1,4 @@ #!/usr/bin/env python -import os from app import notify_celery, create_app - -application = create_app(os.getenv('NOTIFY_API_ENVIRONMENT') or 'development') +application = create_app() application.app_context().push() diff --git a/scripts/run_celery.sh b/scripts/run_celery.sh new file mode 100755 index 000000000..9319fe9f4 --- /dev/null +++ b/scripts/run_celery.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +set -e + +source environment.sh +celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=1 --autoreload -Q sms \ No newline at end of file diff --git a/tests/app/notifications/test_rest.py b/tests/app/notifications/test_rest.py index 326b1d72d..d9f2d7f9c 100644 --- a/tests/app/notifications/test_rest.py +++ b/tests/app/notifications/test_rest.py @@ -30,7 +30,6 @@ def test_get_notification_by_id(notify_api, sample_notification): def test_get_notifications_empty_result(notify_api, sample_api_key): with notify_api.test_request_context(): with notify_api.test_client() as client: - missing_notification_id = uuid.uuid4() auth_header = create_authorization_header( service_id=sample_api_key.service_id, @@ -222,7 +221,8 @@ def test_should_allow_valid_sms_notification(notify_api, sample_template, mocker app.celery.tasks.send_sms.apply_async.assert_called_once_with( (str(sample_template.service_id), notification_id, - "something_encrypted") + "something_encrypted"), + queue="sms" ) assert response.status_code == 201 assert notification_id From 95cdfe08d12ce5b68a6078e41eaac4483d00ea1f Mon Sep 17 00:00:00 2001 From: Martyn Inglis Date: Wed, 17 Feb 2016 11:18:27 +0000 Subject: [PATCH 14/17] Updated logging in run-celery script to log to application logs - poll every second - visibility of 1 minute --- app/celery/tasks.py | 2 +- config.py | 6 +++--- scripts/run_celery.sh | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/celery/tasks.py b/app/celery/tasks.py index 8806ec6ed..3b095d062 100644 --- a/app/celery/tasks.py +++ b/app/celery/tasks.py @@ -1,4 +1,4 @@ -from app import notify_celery, twilio_client, db, encryption +from app import notify_celery, twilio_client, encryption from app.clients.sms.twilio import TwilioClientException from app.dao.templates_dao import get_model_templates from app.dao.notifications_dao import save_notification diff --git a/config.py b/config.py index a399e2883..241791ec3 100644 --- a/config.py +++ b/config.py @@ -23,9 +23,9 @@ class Config(object): BROKER_URL = 'sqs://' BROKER_TRANSPORT_OPTIONS = { 'region': 'eu-west-1', - 'polling_interval': 10, # 1 second - 'visibility_timeout': 3600, # 1 hour - 'queue_name_prefix': 'preview' + 'polling_interval': 1, # 1 second + 'visibility_timeout': 60, # 60 seconds + 'queue_name_prefix': os.environ['NOTIFICATION_QUEUE_PREFIX'] } CELERY_ENABLE_UTC = True, CELERY_TIMEZONE = 'Europe/London' diff --git a/scripts/run_celery.sh b/scripts/run_celery.sh index 9319fe9f4..8e670ea88 100755 --- a/scripts/run_celery.sh +++ b/scripts/run_celery.sh @@ -3,4 +3,4 @@ set -e source environment.sh -celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=1 --autoreload -Q sms \ No newline at end of file +celery -A run_celery.notify_celery worker --loglevel=INFO --logfile=/var/log/notify/application.log --concurrency=4 -Q sms \ No newline at end of file From cebca61f8e1bfc8d72ecb485d88e004adfd3caa9 Mon Sep 17 00:00:00 2001 From: Martyn Inglis Date: Wed, 17 Feb 2016 11:21:49 +0000 Subject: [PATCH 15/17] Travis deploy group for delivery version of api --- .travis.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.travis.yml b/.travis.yml index 446eb43c4..7362b610f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,6 +39,16 @@ deploy: deployment_group: notifications_api_deployment_group region: eu-west-1 on: *2 +- provider: codedeploy + access_key_id: AKIAJQPPNM6P6V53SWKA + secret_access_key: *1 + bucket: notifications-api-codedeploy + key: notifications-api-$TRAVIS_BRANCH-$TRAVIS_BUILD_NUMBER-$TRAVIS_COMMIT.zip + bundle_type: zip + application: notifications-api + deployment_group: notifications-api-celery + region: eu-west-1 + on: *2 before_deploy: - ./scripts/update_version_file.sh - zip -r notifications-api * From e59e66fa21e1abde1a6c96423951f6c32b3f8cac Mon Sep 17 00:00:00 2001 From: Martyn Inglis Date: Wed, 17 Feb 2016 11:25:57 +0000 Subject: [PATCH 16/17] Run celery needed to read in secrets --- run_celery.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/run_celery.py b/run_celery.py index 430577774..814e07f4b 100644 --- a/run_celery.py +++ b/run_celery.py @@ -1,4 +1,12 @@ #!/usr/bin/env python from app import notify_celery, create_app +from credstash import getAllSecrets +import os + +# on aws get secrets and export to env +secrets = getAllSecrets(region="eu-west-1") +for key, val in secrets.items(): + os.environ[key] = val + application = create_app() application.app_context().push() From 58ae041b64985063928f8713afb82fa019984e8c Mon Sep 17 00:00:00 2001 From: Martyn Inglis Date: Wed, 17 Feb 2016 11:36:14 +0000 Subject: [PATCH 17/17] AWS scripts for celery --- aws_run_celery.py | 12 ++++++++++++ run_celery.py | 7 ------- 2 files changed, 12 insertions(+), 7 deletions(-) create mode 100644 aws_run_celery.py diff --git a/aws_run_celery.py b/aws_run_celery.py new file mode 100644 index 000000000..814e07f4b --- /dev/null +++ b/aws_run_celery.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python +from app import notify_celery, create_app +from credstash import getAllSecrets +import os + +# on aws get secrets and export to env +secrets = getAllSecrets(region="eu-west-1") +for key, val in secrets.items(): + os.environ[key] = val + +application = create_app() +application.app_context().push() diff --git a/run_celery.py b/run_celery.py index 814e07f4b..51ce92910 100644 --- a/run_celery.py +++ b/run_celery.py @@ -1,12 +1,5 @@ #!/usr/bin/env python from app import notify_celery, create_app -from credstash import getAllSecrets -import os - -# on aws get secrets and export to env -secrets = getAllSecrets(region="eu-west-1") -for key, val in secrets.items(): - os.environ[key] = val application = create_app() application.app_context().push()