Move job processing into celery

- brings boto S3 into new AWS folder
- CSV processing utils method

Rejigs the jobs rest endpoint - removes some now unused endpoints,

Calls to the task with the job, job processing in task, delegating SMS calls to the sms task

app/__init__.py

@@ -1,3 +1,4 @@
+import uuid
 import os
 import re
 from flask import request, url_for
@@ -50,7 +51,7 @@ def create_app():
     application.register_blueprint(user_blueprint, url_prefix='/user')
     application.register_blueprint(template_blueprint)
     application.register_blueprint(status_blueprint, url_prefix='/status')
-    application.register_blueprint(notifications_blueprint, url_prefix='/notifications')
+    application.register_blueprint(notifications_blueprint)
     application.register_blueprint(job_blueprint)
 
     return application
@@ -99,3 +100,7 @@ def email_safe(string):
         character.lower() if character.isalnum() or character == "." else ""
         for character in re.sub("\s+", ".", string.strip())
     ])
+
+
+def create_uuid():
+    return str(uuid.uuid4())

app/aws/s3.py

@@ -0,0 +1,7 @@
+from boto3 import resource
+
+
+def get_job_from_s3(bucket_name, job_id):
+    s3 = resource('s3')
+    key = s3.Object(bucket_name, '{}.csv'.format(job_id))
+    return key.get()['Body'].read().decode('utf-8')

app/celery/tasks.py

@@ -1,11 +1,42 @@
+from app import create_uuid
 from app import notify_celery, encryption, firetext_client, aws_ses_client
 from app.clients.email.aws_ses import AwsSesClientException
 from app.clients.sms.firetext import FiretextClientException
 from app.dao.templates_dao import dao_get_template_by_id
 from app.dao.notifications_dao import save_notification
+from app.dao.jobs_dao import dao_update_job, dao_get_job_by_id
 from app.models import Notification
 from flask import current_app
 from sqlalchemy.exc import SQLAlchemyError
+from app.aws import s3
+from app.csv import get_mobile_numbers_from_csv
+
+
+@notify_celery.task(name="process-job")
+def process_job(job_id):
+    job = dao_get_job_by_id(job_id)
+    job.status = 'in progress'
+    dao_update_job(job)
+
+    file = s3.get_job_from_s3(job.bucket_name, job_id)
+    mobile_numbers = get_mobile_numbers_from_csv(file)
+
+    for mobile_number in mobile_numbers:
+        notification = encryption.encrypt({
+            'template': job.template_id,
+            'job': str(job.id),
+            'to': mobile_number
+        })
+
+        send_sms.apply_async((
+            str(job.service_id),
+            str(create_uuid()),
+            notification),
+            queue='sms'
+        )
+
+    job.status = 'finished'
+    dao_update_job(job)
 
 
 @notify_celery.task(name="send-sms")

app/csv.py

@@ -0,0 +1,12 @@
+import csv
+
+
+def get_mobile_numbers_from_csv(file_data):
+    numbers = []
+    reader = csv.DictReader(
+        file_data.splitlines(),
+        lineterminator='\n',
+        quoting=csv.QUOTE_NONE)
+    for i, row in enumerate(reader):
+        numbers.append(row['phone'].replace(' ', ''))
+    return numbers

app/dao/jobs_dao.py

@@ -2,24 +2,23 @@ from app import db
 from app.models import Job
 
 
-def save_job(job, update_dict={}):
+def dao_get_job_by_service_id_and_job_id(service_id, job_id):
-    if update_dict:
-        update_dict.pop('id', None)
-        update_dict.pop('service', None)
-        update_dict.pop('template', None)
-        Job.query.filter_by(id=job.id).update(update_dict)
-    else:
-        db.session.add(job)
-    db.session.commit()
-
-
-def get_job(service_id, job_id):
     return Job.query.filter_by(service_id=service_id, id=job_id).first()
 
 
-def get_jobs_by_service(service_id):
+def dao_get_jobs_by_service_id(service_id):
     return Job.query.filter_by(service_id=service_id).all()
 
 
-def _get_jobs():
+def dao_get_job_by_id(job_id):
-    return Job.query.all()
+    return Job.query.filter_by(id=job_id).first()
+
+
+def dao_create_job(job):
+    db.session.add(job)
+    db.session.commit()
+
+
+def dao_update_job(job):
+    db.session.add(job)
+    db.session.commit()

app/errors.py

@@ -43,7 +43,7 @@ def register_errors(blueprint):
     @blueprint.app_errorhandler(DataError)
     def no_result_found(e):
         current_app.logger.error(e)
-        return jsonify(error="No result found"), 404
+        return jsonify(result="error", message="No result found"), 404
 
     @blueprint.app_errorhandler(SQLAlchemyError)
     def db_error(e):

app/job/rest.py

@@ -1,148 +1,81 @@
-import boto3
-import json
-
 from flask import (
     Blueprint,
     jsonify,
-    request,
+    request
-    current_app
 )
 
-from sqlalchemy.exc import DataError
-from sqlalchemy.orm.exc import NoResultFound
-
 from app.dao.jobs_dao import (
-    save_job,
+    dao_create_job,
-    get_job,
+    dao_get_job_by_service_id_and_job_id,
-    get_jobs_by_service
+    dao_get_jobs_by_service_id,
+    dao_update_job
 )
 
-from app.dao import notifications_dao
+from app.dao.services_dao import (
+    dao_fetch_service_by_id
+)
 
 from app.schemas import (
     job_schema,
-    jobs_schema,
+    jobs_schema
-    job_schema_load_json,
-    notification_status_schema,
-    notifications_status_schema,
-    notification_status_schema_load_json
 )
 
+from app.celery.tasks import process_job
+
 job = Blueprint('job', __name__, url_prefix='/service/<service_id>/job')
 
-
 from app.errors import register_errors
+
 register_errors(job)
 
 
 @job.route('/<job_id>', methods=['GET'])
+def get_job_by_service_and_job_id(service_id, job_id):
+    job = dao_get_job_by_service_id_and_job_id(service_id, job_id)
+    if not job:
+        return jsonify(result="error", message="Job {} not found for service {}".format(job_id, service_id)), 404
+    data, errors = job_schema.dump(job)
+    return jsonify(data=data)
+
+
 @job.route('', methods=['GET'])
-def get_job_for_service(service_id, job_id=None):
+def get_jobs_by_service(service_id):
-    if job_id:
+    jobs = dao_get_jobs_by_service_id(service_id)
-        try:
+    data, errors = jobs_schema.dump(jobs)
-            job = get_job(service_id, job_id)
+    return jsonify(data=data)
-            if not job:
-                return jsonify(result="error", message="Job not found"), 404
-            data, errors = job_schema.dump(job)
-            return jsonify(data=data)
-        except DataError:
-            return jsonify(result="error", message="Invalid job id"), 400
-    else:
-        jobs = get_jobs_by_service(service_id)
-        data, errors = jobs_schema.dump(jobs)
-        return jsonify(data=data)
 
 
 @job.route('', methods=['POST'])
 def create_job(service_id):
-    job, errors = job_schema.load(request.get_json())
+
+    service = dao_fetch_service_by_id(service_id)
+    if not service:
+        return jsonify(result="error", message="Service {} not found".format(service_id)), 404
+
+    data = request.get_json()
+    data.update({
+        "service": service_id
+    })
+    job, errors = job_schema.load(data)
     if errors:
         return jsonify(result="error", message=errors), 400
 
-    save_job(job)
+    dao_create_job(job)
-    _enqueue_job(job)
+    process_job.apply_async([str(job.id)], queue="process-job")
-
     return jsonify(data=job_schema.dump(job).data), 201
 
 
-@job.route('/<job_id>', methods=['PUT'])
+@job.route('/<job_id>', methods=['POST'])
 def update_job(service_id, job_id):
+    fetched_job = dao_get_job_by_service_id_and_job_id(service_id, job_id)
+    if not fetched_job:
+        return jsonify(result="error", message="Job {} not found for service {}".format(job_id, service_id)), 404
 
-    job = get_job(service_id, job_id)
+    current_data = dict(job_schema.dump(fetched_job).data.items())
-    update_dict, errors = job_schema_load_json.load(request.get_json())
+    current_data.update(request.get_json())
+
+    update_dict, errors = job_schema.load(current_data)
     if errors:
         return jsonify(result="error", message=errors), 400
-
+    dao_update_job(update_dict)
-    save_job(job, update_dict=update_dict)
+    return jsonify(data=job_schema.dump(update_dict).data), 200
-
-    return jsonify(data=job_schema.dump(job).data), 200
-
-
-@job.route('/<job_id>/notification', methods=['POST'])
-def create_notification_for_job(service_id, job_id):
-
-    # TODO assert service_id == payload service id
-    # and same for job id
-    notification, errors = notification_status_schema.load(request.get_json())
-    if errors:
-        return jsonify(result="error", message=errors), 400
-
-    notifications_dao.save_notification(notification)
-
-    return jsonify(data=notification_status_schema.dump(notification).data), 201
-
-
-@job.route('/<job_id>/notification', methods=['GET'])
-@job.route('/<job_id>/notification/<notification_id>')
-def get_notification_for_job(service_id, job_id, notification_id=None):
-    if notification_id:
-        try:
-            notification = notifications_dao.get_notification_for_job(service_id, job_id, notification_id)
-            data, errors = notification_status_schema.dump(notification)
-            return jsonify(data=data)
-        except DataError:
-            return jsonify(result="error", message="Invalid notification id"), 400
-        except NoResultFound:
-            return jsonify(result="error", message="Notification not found"), 404
-    else:
-        notifications = notifications_dao.get_notifications_for_job(service_id, job_id)
-        data, errors = notifications_status_schema.dump(notifications)
-        return jsonify(data=data)
-
-
-@job.route('/<job_id>/notification/<notification_id>', methods=['PUT'])
-def update_notification_for_job(service_id, job_id, notification_id):
-
-    notification = notifications_dao.get_notification_for_job(service_id, job_id, notification_id)
-    update_dict, errors = notification_status_schema_load_json.load(request.get_json())
-
-    if errors:
-        return jsonify(result="error", message=errors), 400
-
-    notifications_dao.save_notification(notification, update_dict=update_dict)
-
-    return jsonify(data=job_schema.dump(notification).data), 200
-
-
-def _enqueue_job(job):
-    aws_region = current_app.config['AWS_REGION']
-    queue_name = current_app.config['NOTIFY_JOB_QUEUE']
-
-    queue = boto3.resource('sqs', region_name=aws_region).create_queue(QueueName=queue_name)
-    data = {
-        'id': str(job.id),
-        'service': str(job.service.id),
-        'template': job.template.id,
-        'bucket_name': job.bucket_name,
-        'file_name': job.file_name,
-        'original_file_name': job.original_file_name
-    }
-    job_json = json.dumps(data)
-    queue.send_message(MessageBody=job_json,
-                       MessageAttributes={'id': {'StringValue': str(job.id), 'DataType': 'String'},
-                                          'service': {'StringValue': str(job.service.id), 'DataType': 'String'},
-                                          'template': {'StringValue': str(job.template.id), 'DataType': 'String'},
-                                          'bucket_name': {'StringValue': job.bucket_name, 'DataType': 'String'},
-                                          'file_name': {'StringValue': job.file_name, 'DataType': 'String'},
-                                          'original_file_name': {'StringValue': job.original_file_name,
-                                                                 'DataType': 'String'}})

app/notifications/rest.py

@@ -1,5 +1,3 @@
-import uuid
-
 from flask import (
     Blueprint,
     jsonify,
@@ -7,7 +5,7 @@ from flask import (
     current_app
 )
 
-from app import api_user, encryption
+from app import api_user, encryption, create_uuid
 from app.dao import (
     templates_dao,
     services_dao,
@@ -34,11 +32,7 @@ SMS_NOTIFICATION = 'sms'
 EMAIL_NOTIFICATION = 'email'
 
 
-def create_notification_id():
+@notifications.route('/notifications/<string:notification_id>', methods=['GET'])
-    return str(uuid.uuid4())
-
-
-@notifications.route('/<string:notification_id>', methods=['GET'])
 def get_notifications(notification_id):
     try:
         notification = notifications_dao.get_notification(api_user['client'], notification_id)
@@ -47,22 +41,22 @@ def get_notifications(notification_id):
         return jsonify(result="error", message="not found"), 404
 
 
-@notifications.route('/sms', methods=['POST'])
+@notifications.route('/notifications/sms', methods=['POST'])
 def create_sms_notification():
     return send_notification(notification_type=SMS_NOTIFICATION, expects_job=False)
 
 
-@notifications.route('/sms/service/<service_id>', methods=['POST'])
+@notifications.route('/notifications/sms/service/<service_id>', methods=['POST'])
 def create_sms_for_job(service_id):
     return send_notification(service_id=service_id, notification_type=SMS_NOTIFICATION, expects_job=True)
 
 
-@notifications.route('/email', methods=['POST'])
+@notifications.route('/notifications/email', methods=['POST'])
 def create_email_notification():
     return send_notification(notification_type=EMAIL_NOTIFICATION, expects_job=False)
 
 
-@notifications.route('/email/service/<service_id>', methods=['POST'])
+@notifications.route('/notifications/email/service/<service_id>', methods=['POST'])
 def create_email_notification_for_job(service_id):
     return send_notification(service_id=service_id, notification_type=EMAIL_NOTIFICATION, expects_job=True)
 
@@ -98,7 +92,7 @@ def send_notification(notification_type, service_id=None, expects_job=False):
         ), 400
 
     if expects_job:
-        job = jobs_dao.get_job(service_id, notification['job'])
+        job = jobs_dao.dao_get_job_by_service_id_and_job_id(service_id, notification['job'])
 
         if not job:
             return jsonify(result="error", message={'job': ['Job {} not found'.format(notification['job'])]}), 400
@@ -115,7 +109,7 @@ def send_notification(notification_type, service_id=None, expects_job=False):
                 return jsonify(
                     result="error", message={'to': ['Email address not permitted for restricted service']}), 400
 
-    notification_id = create_notification_id()
+    notification_id = create_uuid()
 
     if notification_type is SMS_NOTIFICATION:
         send_sms.apply_async((

scripts/run_celery.sh

@@ -3,4 +3,4 @@
 set -e
 
 source environment.sh
-celery -A run_celery.notify_celery worker --loglevel=INFO --logfile=/var/log/notify/application.log --concurrency=4 -Q sms,sms-code,email-code,email
+celery -A run_celery.notify_celery worker --loglevel=INFO --logfile=/var/log/notify/application.log --concurrency=4 -Q sms,sms-code,email-code,email,process-job

test_csv_files/multiple_sms.csv

@@ -0,0 +1,11 @@
+phone
++441234123121
++441234123122
++441234123123
++441234123124
++441234123125
++441234123126
++441234123127
++441234123128
++441234123129
++441234123120

test_csv_files/sms.csv

@@ -0,0 +1,2 @@
+phone
++441234123123

tests/app/__init__.py

@@ -0,0 +1,7 @@
+import os
+
+
+def load_example_csv(file):
+    file_path = os.path.join("test_csv_files", "{}.csv".format(file))
+    with open(file_path) as f:
+        return f.read()

tests/app/celery/test_tasks.py

@@ -1,13 +1,49 @@
 import uuid
 import pytest
 from flask import current_app
-from app.celery.tasks import (send_sms, send_sms_code, send_email_code, send_email)
+from app.celery.tasks import (send_sms, send_sms_code, send_email_code, send_email, process_job)
 from app import (firetext_client, aws_ses_client, encryption)
 from app.clients.email.aws_ses import AwsSesClientException
 from app.clients.sms.firetext import FiretextClientException
-from app.dao import notifications_dao
+from app.dao import notifications_dao, jobs_dao
 from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy.orm.exc import NoResultFound
+from app.celery.tasks import s3
+from app.celery import tasks
+from tests.app import load_example_csv
+
+
+def test_should_process_sms_job(sample_job, mocker):
+    mocker.patch('app.celery.tasks.s3.get_job_from_s3', return_value=load_example_csv('sms'))
+    mocker.patch('app.celery.tasks.send_sms.apply_async')
+    mocker.patch('app.encryption.encrypt', return_value="something_encrypted")
+    mocker.patch('app.celery.tasks.create_uuid', return_value="uuid")
+
+    process_job(sample_job.id)
+
+    s3.get_job_from_s3.assert_called_once_with(sample_job.bucket_name, sample_job.id)
+    tasks.send_sms.apply_async.assert_called_once_with(
+        (str(sample_job.service_id),
+         "uuid",
+         "something_encrypted"),
+        queue="sms"
+    )
+    job = jobs_dao.dao_get_job_by_id(sample_job.id)
+    assert job.status == 'finished'
+
+
+def test_should_process_all_sms_job(sample_job, mocker):
+    mocker.patch('app.celery.tasks.s3.get_job_from_s3', return_value=load_example_csv('multiple_sms'))
+    mocker.patch('app.celery.tasks.send_sms.apply_async')
+    mocker.patch('app.encryption.encrypt', return_value="something_encrypted")
+    mocker.patch('app.celery.tasks.create_uuid', return_value="uuid")
+
+    process_job(sample_job.id)
+
+    s3.get_job_from_s3.assert_called_once_with(sample_job.bucket_name, sample_job.id)
+    tasks.send_sms.apply_async.call_count == 10
+    job = jobs_dao.dao_get_job_by_id(sample_job.id)
+    assert job.status == 'finished'
 
 
 def test_should_send_template_to_correct_sms_provider_and_persist(sample_template, mocker):
@@ -218,7 +254,9 @@ def test_should_send_email_code(mocker):
 
     send_email_code(encrypted_verification)
 
-    aws_ses_client.send_email.assert_called_once_with(current_app.config['VERIFY_CODE_FROM_EMAIL_ADDRESS'],
+    aws_ses_client.send_email.assert_called_once_with(
-                                                      verification['to'],
+        current_app.config['VERIFY_CODE_FROM_EMAIL_ADDRESS'],
-                                                      "Verification code",
+        verification['to'],
-                                                      verification['secret_code'])
+        "Verification code",
+        verification['secret_code']
+    )

tests/app/conftest.py

@@ -6,7 +6,7 @@ from app.dao.users_dao import (save_model_user, create_user_code, create_secret_
 from app.dao.services_dao import dao_create_service
 from app.dao.templates_dao import dao_create_template
 from app.dao.api_key_dao import save_model_api_key
-from app.dao.jobs_dao import save_job
+from app.dao.jobs_dao import dao_create_job
 from app.dao.notifications_dao import save_notification
 import uuid
 
@@ -204,7 +204,7 @@ def sample_job(notify_db,
         'notification_count': 1
     }
     job = Job(**data)
-    save_job(job)
+    dao_create_job(job)
     return job
 
 

tests/app/dao/test_jobs_dao.py

@@ -1,18 +1,16 @@
 import uuid
-import json
 
 from app.dao.jobs_dao import (
-    save_job,
+    dao_get_job_by_service_id_and_job_id,
-    get_job,
+    dao_create_job,
-    get_jobs_by_service,
+    dao_update_job,
-    _get_jobs
+    dao_get_jobs_by_service_id
 )
 
 from app.models import Job
 
 
-def test_save_job(notify_db, notify_db_session, sample_template):
+def test_create_job(sample_template):
-
     assert Job.query.count() == 0
 
     job_id = uuid.uuid4()
@@ -29,39 +27,33 @@ def test_save_job(notify_db, notify_db_session, sample_template):
     }
 
     job = Job(**data)
-    save_job(job)
+    dao_create_job(job)
 
     assert Job.query.count() == 1
     job_from_db = Job.query.get(job_id)
     assert job == job_from_db
 
 
-def test_get_job_by_id(notify_db, notify_db_session, sample_job):
+def test_get_job_by_id(sample_job):
-    job_from_db = get_job(sample_job.service.id, sample_job.id)
+    job_from_db = dao_get_job_by_service_id_and_job_id(sample_job.service.id, sample_job.id)
     assert sample_job == job_from_db
 
 
 def test_get_jobs_for_service(notify_db, notify_db_session, sample_template):
-
     from tests.app.conftest import sample_job as create_job
     from tests.app.conftest import sample_service as create_service
     from tests.app.conftest import sample_template as create_template
     from tests.app.conftest import sample_user as create_user
 
-    one_job = create_job(notify_db, notify_db_session, sample_template.service,
+    one_job = create_job(notify_db, notify_db_session, sample_template.service, sample_template)
-                         sample_template)
 
-    other_user = create_user(notify_db, notify_db_session,
+    other_user = create_user(notify_db, notify_db_session, email="test@digital.cabinet-office.gov.uk")
-                             email="test@digital.cabinet-office.gov.uk")
+    other_service = create_service(notify_db, notify_db_session, user=other_user, service_name="other service")
-    other_service = create_service(notify_db, notify_db_session,
+    other_template = create_template(notify_db, notify_db_session, service=other_service)
-                                   user=other_user, service_name="other service")
+    other_job = create_job(notify_db, notify_db_session, service=other_service, template=other_template)
-    other_template = create_template(notify_db, notify_db_session,
-                                     service=other_service)
-    other_job = create_job(notify_db, notify_db_session, service=other_service,
-                           template=other_template)
 
-    one_job_from_db = get_jobs_by_service(one_job.service_id)
+    one_job_from_db = dao_get_jobs_by_service_id(one_job.service_id)
-    other_job_from_db = get_jobs_by_service(other_job.service_id)
+    other_job_from_db = dao_get_jobs_by_service_id(other_job.service_id)
 
     assert len(one_job_from_db) == 1
     assert one_job == one_job_from_db[0]
@@ -72,31 +64,12 @@ def test_get_jobs_for_service(notify_db, notify_db_session, sample_template):
     assert one_job_from_db != other_job_from_db
 
 
-def test_get_all_jobs(notify_db, notify_db_session, sample_template):
+def test_update_job(sample_job):
-    from tests.app.conftest import sample_job as create_job
-    for i in range(5):
-        create_job(notify_db,
-                   notify_db_session,
-                   sample_template.service,
-                   sample_template)
-    jobs_from_db = _get_jobs()
-    assert len(jobs_from_db) == 5
-
-
-def test_update_job(notify_db, notify_db_session, sample_job):
     assert sample_job.status == 'pending'
 
-    update_dict = {
+    sample_job.status = 'in progress'
-        'id': sample_job.id,
-        'service': sample_job.service.id,
-        'template': sample_job.template.id,
-        'bucket_name': sample_job.bucket_name,
-        'file_name': sample_job.file_name,
-        'original_file_name': sample_job.original_file_name,
-        'status': 'in progress'
-    }
 
-    save_job(sample_job, update_dict=update_dict)
+    dao_update_job(sample_job)
 
     job_from_db = Job.query.get(sample_job.id)
 

tests/app/dao/test_notification_dao.py

@@ -8,7 +8,7 @@ from app.dao.notifications_dao import (
 )
 
 
-def test_save_notification(notify_db, notify_db_session, sample_template, sample_job):
+def test_save_notification(sample_template, sample_job):
 
     assert Notification.query.count() == 0
     to = '+44709123456'

tests/app/job/test_job_rest.py

@@ -1,328 +0,0 @@
-import boto3
-import moto
-import json
-import uuid
-from flask import url_for
-
-from tests import create_authorization_header
-from tests.app.conftest import sample_job as create_job
-
-
-def test_get_jobs(notify_api, notify_db, notify_db_session, sample_template):
-    _setup_jobs(notify_db, notify_db_session, sample_template)
-
-    service_id = sample_template.service.id
-
-    with notify_api.test_request_context():
-        with notify_api.test_client() as client:
-            path = url_for('job.get_job_for_service', service_id=service_id)
-            auth_header = create_authorization_header(service_id=service_id,
-                                                      path=path,
-                                                      method='GET')
-            response = client.get(path, headers=[auth_header])
-            assert response.status_code == 200
-            resp_json = json.loads(response.get_data(as_text=True))
-            assert len(resp_json['data']) == 5
-
-
-def test_get_job_with_invalid_id_returns400(notify_api, notify_db,
-                                            notify_db_session,
-                                            sample_template):
-    service_id = sample_template.service.id
-    with notify_api.test_request_context():
-        with notify_api.test_client() as client:
-            path = url_for('job.get_job_for_service', job_id='invalid_id', service_id=service_id)
-            auth_header = create_authorization_header(service_id=sample_template.service.id,
-                                                      path=path,
-                                                      method='GET')
-            response = client.get(path, headers=[auth_header])
-            assert response.status_code == 400
-            resp_json = json.loads(response.get_data(as_text=True))
-            assert resp_json == {'message': 'Invalid job id',
-                                 'result': 'error'}
-
-
-def test_get_job_with_unknown_id_returns404(notify_api, notify_db,
-                                            notify_db_session,
-                                            sample_template):
-    random_id = str(uuid.uuid4())
-    service_id = sample_template.service.id
-    with notify_api.test_request_context():
-        with notify_api.test_client() as client:
-            path = url_for('job.get_job_for_service', job_id=random_id, service_id=service_id)
-            auth_header = create_authorization_header(service_id=sample_template.service.id,
-                                                      path=path,
-                                                      method='GET')
-            response = client.get(path, headers=[auth_header])
-            assert response.status_code == 404
-            resp_json = json.loads(response.get_data(as_text=True))
-            assert resp_json == {'message': 'Job not found', 'result': 'error'}
-
-
-def test_get_job_by_id(notify_api, notify_db, notify_db_session,
-                       sample_job):
-    job_id = str(sample_job.id)
-    service_id = sample_job.service.id
-    with notify_api.test_request_context():
-        with notify_api.test_client() as client:
-            path = url_for('job.get_job_for_service', job_id=job_id, service_id=service_id)
-            auth_header = create_authorization_header(service_id=sample_job.service.id,
-                                                      path=path,
-                                                      method='GET')
-            response = client.get(path, headers=[auth_header])
-            assert response.status_code == 200
-            resp_json = json.loads(response.get_data(as_text=True))
-            assert resp_json['data']['id'] == job_id
-
-
-@moto.mock_sqs
-def test_create_job(notify_api, notify_db, notify_db_session, sample_template):
-    job_id = uuid.uuid4()
-    template_id = sample_template.id
-    service_id = sample_template.service.id
-    original_file_name = 'thisisatest.csv'
-    bucket_name = 'service-{}-notify'.format(service_id)
-    file_name = '{}.csv'.format(job_id)
-    data = {
-        'id': str(job_id),
-        'service': str(service_id),
-        'template': template_id,
-        'original_file_name': original_file_name,
-        'bucket_name': bucket_name,
-        'file_name': file_name,
-        'notification_count': 1
-    }
-
-    with notify_api.test_request_context():
-        with notify_api.test_client() as client:
-            path = url_for('job.create_job', service_id=service_id)
-            auth_header = create_authorization_header(service_id=sample_template.service.id,
-                                                      path=path,
-                                                      method='POST',
-                                                      request_body=json.dumps(data))
-            headers = [('Content-Type', 'application/json'), auth_header]
-            response = client.post(
-                path,
-                data=json.dumps(data),
-                headers=headers)
-    assert response.status_code == 201
-
-    resp_json = json.loads(response.get_data(as_text=True))
-
-    assert resp_json['data']['id'] == str(job_id)
-    assert resp_json['data']['service'] == str(service_id)
-    assert resp_json['data']['template'] == template_id
-    assert resp_json['data']['original_file_name'] == original_file_name
-
-    boto3.setup_default_session(region_name='eu-west-1')
-    q = boto3.resource('sqs').get_queue_by_name(QueueName=notify_api.config['NOTIFY_JOB_QUEUE'])
-    messages = q.receive_messages()
-    assert len(messages) == 1
-
-    expected_message = json.loads(messages[0].body)
-    assert expected_message['id'] == str(job_id)
-    assert expected_message['service'] == str(service_id)
-    assert expected_message['template'] == template_id
-    assert expected_message['bucket_name'] == bucket_name
-
-
-def test_get_update_job_status(notify_api,
-                               notify_db,
-                               notify_db_session,
-                               sample_job):
-
-    assert sample_job.status == 'pending'
-
-    job_id = str(sample_job.id)
-    service_id = str(sample_job.service.id)
-
-    update_data = {
-        'id': job_id,
-        'service': service_id,
-        'template': sample_job.template.id,
-        'bucket_name': sample_job.bucket_name,
-        'file_name': sample_job.file_name,
-        'original_file_name': sample_job.original_file_name,
-        'status': 'in progress',
-        'notification_count': 1
-    }
-
-    with notify_api.test_request_context():
-        with notify_api.test_client() as client:
-            path = url_for('job.update_job', service_id=service_id, job_id=job_id)
-
-            auth_header = create_authorization_header(service_id=service_id,
-                                                      path=path,
-                                                      method='PUT',
-                                                      request_body=json.dumps(update_data))
-
-            headers = [('Content-Type', 'application/json'), auth_header]
-
-            response = client.put(path, headers=headers, data=json.dumps(update_data))
-
-            assert response.status_code == 200
-            resp_json = json.loads(response.get_data(as_text=True))
-            assert resp_json['data']['status'] == 'in progress'
-
-
-def test_get_notification(notify_api, notify_db, notify_db_session, sample_notification):
-
-    with notify_api.test_request_context():
-        with notify_api.test_client() as client:
-            path = url_for('job.get_notification_for_job',
-                           service_id=sample_notification.service.id,
-                           job_id=sample_notification.job.id,
-                           notification_id=sample_notification.id)
-
-            auth_header = create_authorization_header(service_id=sample_notification.service.id,
-                                                      path=path,
-                                                      method='GET')
-
-            headers = [('Content-Type', 'application/json'), auth_header]
-            response = client.get(path, headers=headers)
-            resp_json = json.loads(response.get_data(as_text=True))
-
-            assert str(sample_notification.id) == resp_json['data']['id']
-            assert str(sample_notification.service.id) == resp_json['data']['service']
-            assert sample_notification.template.id == resp_json['data']['template']
-            assert str(sample_notification.job.id) == resp_json['data']['job']
-            assert sample_notification.status == resp_json['data']['status']
-
-
-def test_get_notifications(notify_api, notify_db, notify_db_session, sample_job):
-
-    from tests.app.conftest import sample_notification
-    for i in range(0, 5):
-        sample_notification(notify_db,
-                            notify_db_session,
-                            service=sample_job.service,
-                            template=sample_job.template,
-                            job=sample_job)
-
-    service_id = str(sample_job.service.id)
-    job_id = str(sample_job.id)
-
-    with notify_api.test_request_context():
-        with notify_api.test_client() as client:
-            path = url_for('job.get_notification_for_job',
-                           service_id=service_id,
-                           job_id=job_id)
-
-            auth_header = create_authorization_header(service_id=service_id,
-                                                      path=path,
-                                                      method='GET')
-
-            headers = [('Content-Type', 'application/json'), auth_header]
-            response = client.get(path, headers=headers)
-            resp_json = json.loads(response.get_data(as_text=True))
-
-            assert len(resp_json['data']) == 5
-
-
-def test_add_notification(notify_api, notify_db, notify_db_session, sample_job):
-
-    to = '+44709123456'
-    data = {
-        'to': to,
-        'job': str(sample_job.id),
-        'service': str(sample_job.service.id),
-        'template': sample_job.template.id
-    }
-    with notify_api.test_request_context():
-        with notify_api.test_client() as client:
-            path = url_for('job.create_notification_for_job',
-                           service_id=sample_job.service.id,
-                           job_id=sample_job.id)
-
-            auth_header = create_authorization_header(service_id=sample_job.service.id,
-                                                      path=path,
-                                                      method='POST',
-                                                      request_body=json.dumps(data))
-
-            headers = [('Content-Type', 'application/json'), auth_header]
-
-            response = client.post(path, headers=headers, data=json.dumps(data))
-
-            resp_json = json.loads(response.get_data(as_text=True))
-
-            assert resp_json['data']['id']
-            assert data['to'] == resp_json['data']['to']
-            assert data['service'] == resp_json['data']['service']
-            assert data['template'] == resp_json['data']['template']
-            assert data['job'] == resp_json['data']['job']
-            assert 'sent' == resp_json['data']['status']
-
-
-def test_add_notification_with_id(notify_api, notify_db, notify_db_session, sample_job):
-    notification_id = str(uuid.uuid4())
-    to = '+44709123456'
-    data = {
-        'id': notification_id,
-        'to': to,
-        'job': str(sample_job.id),
-        'service': str(sample_job.service.id),
-        'template': sample_job.template.id
-    }
-    with notify_api.test_request_context():
-        with notify_api.test_client() as client:
-            path = url_for('job.create_notification_for_job',
-                           service_id=sample_job.service.id,
-                           job_id=sample_job.id)
-
-            auth_header = create_authorization_header(service_id=sample_job.service.id,
-                                                      path=path,
-                                                      method='POST',
-                                                      request_body=json.dumps(data))
-
-            headers = [('Content-Type', 'application/json'), auth_header]
-
-            response = client.post(path, headers=headers, data=json.dumps(data))
-
-            resp_json = json.loads(response.get_data(as_text=True))
-
-            assert resp_json['data']['id'] == notification_id
-            assert data['to'] == resp_json['data']['to']
-            assert data['service'] == resp_json['data']['service']
-            assert data['template'] == resp_json['data']['template']
-            assert data['job'] == resp_json['data']['job']
-            assert 'sent' == resp_json['data']['status']
-
-
-def test_update_notification(notify_api, notify_db, notify_db_session, sample_notification):
-
-    assert sample_notification.status == 'sent'
-
-    update_data = {
-        'id': str(sample_notification.id),
-        'to': sample_notification.to,
-        'job': str(sample_notification.job.id),
-        'service': str(sample_notification.service.id),
-        'template': sample_notification.template.id,
-        'status': 'failed'
-    }
-    with notify_api.test_request_context():
-        with notify_api.test_client() as client:
-            path = url_for('job.update_notification_for_job',
-                           service_id=sample_notification.service.id,
-                           job_id=sample_notification.job.id,
-                           notification_id=sample_notification.id)
-
-            auth_header = create_authorization_header(service_id=sample_notification.service.id,
-                                                      path=path,
-                                                      method='PUT',
-                                                      request_body=json.dumps(update_data))
-
-            headers = [('Content-Type', 'application/json'), auth_header]
-
-            response = client.put(path, headers=headers, data=json.dumps(update_data))
-
-            resp_json = json.loads(response.get_data(as_text=True))
-
-            assert update_data['id'] == resp_json['data']['id']
-            assert 'failed' == resp_json['data']['status']
-
-
-def _setup_jobs(notify_db, notify_db_session, template, number_of_jobs=5):
-    for i in range(number_of_jobs):
-        create_job(notify_db, notify_db_session, service=template.service,
-                   template=template)

tests/app/job/test_rest.py

@@ -0,0 +1,226 @@
+import json
+import uuid
+import app.celery.tasks
+
+from tests import create_authorization_header
+from tests.app.conftest import sample_job as create_job
+
+
+def test_get_jobs(notify_api, notify_db, notify_db_session, sample_template):
+    _setup_jobs(notify_db, notify_db_session, sample_template)
+
+    service_id = sample_template.service.id
+
+    with notify_api.test_request_context():
+        with notify_api.test_client() as client:
+            path = '/service/{}/job'.format(service_id)
+            auth_header = create_authorization_header(
+                service_id=service_id,
+                path=path,
+                method='GET')
+            response = client.get(path, headers=[auth_header])
+            assert response.status_code == 200
+            resp_json = json.loads(response.get_data(as_text=True))
+            assert len(resp_json['data']) == 5
+
+
+def test_get_job_with_invalid_service_id_returns404(notify_api, sample_api_key, sample_service):
+    with notify_api.test_request_context():
+        with notify_api.test_client() as client:
+            path = '/service/{}/job'.format(sample_service.id)
+            auth_header = create_authorization_header(
+                service_id=sample_service.id,
+                path=path,
+                method='GET')
+            response = client.get(path, headers=[auth_header])
+            assert response.status_code == 200
+            resp_json = json.loads(response.get_data(as_text=True))
+            assert len(resp_json['data']) == 0
+
+
+def test_get_job_with_invalid_job_id_returns404(notify_api, sample_template):
+    service_id = sample_template.service.id
+    with notify_api.test_request_context():
+        with notify_api.test_client() as client:
+            path = '/service/{}/job/{}'.format(service_id, "bad-id")
+            auth_header = create_authorization_header(
+                service_id=sample_template.service.id,
+                path=path,
+                method='GET')
+            response = client.get(path, headers=[auth_header])
+            assert response.status_code == 404
+            resp_json = json.loads(response.get_data(as_text=True))
+            print(resp_json)
+            assert resp_json['result'] == 'error'
+            assert resp_json['message'] == 'No result found'
+
+
+def test_get_job_with_unknown_id_returns404(notify_api, sample_template):
+    random_id = str(uuid.uuid4())
+    service_id = sample_template.service.id
+    with notify_api.test_request_context():
+        with notify_api.test_client() as client:
+            path = '/service/{}/job/{}'.format(service_id, random_id)
+            auth_header = create_authorization_header(
+                service_id=sample_template.service.id,
+                path=path,
+                method='GET')
+            response = client.get(path, headers=[auth_header])
+            assert response.status_code == 404
+            resp_json = json.loads(response.get_data(as_text=True))
+            assert resp_json == {
+                'message': 'Job {} not found for service {}'.format(random_id, service_id),
+                'result': 'error'
+            }
+
+
+def test_get_job_by_id(notify_api, sample_job):
+    job_id = str(sample_job.id)
+    service_id = sample_job.service.id
+    with notify_api.test_request_context():
+        with notify_api.test_client() as client:
+            path = '/service/{}/job/{}'.format(service_id, job_id)
+            auth_header = create_authorization_header(
+                service_id=sample_job.service.id,
+                path=path,
+                method='GET')
+            response = client.get(path, headers=[auth_header])
+            assert response.status_code == 200
+            resp_json = json.loads(response.get_data(as_text=True))
+            assert resp_json['data']['id'] == job_id
+
+
+def test_create_job(notify_api, sample_template, mocker):
+    with notify_api.test_request_context():
+        with notify_api.test_client() as client:
+            mocker.patch('app.celery.tasks.process_job.apply_async')
+            job_id = uuid.uuid4()
+            data = {
+                'id': str(job_id),
+                'service': str(sample_template.service.id),
+                'template': sample_template.id,
+                'original_file_name': 'thisisatest.csv',
+                'bucket_name': 'service-{}-notify'.format(sample_template.service.id),
+                'file_name': '{}.csv'.format(job_id),
+                'notification_count': 1
+            }
+            path = '/service/{}/job'.format(sample_template.service.id)
+            auth_header = create_authorization_header(
+                service_id=sample_template.service.id,
+                path=path,
+                method='POST',
+                request_body=json.dumps(data))
+            headers = [('Content-Type', 'application/json'), auth_header]
+            response = client.post(
+                path,
+                data=json.dumps(data),
+                headers=headers)
+            assert response.status_code == 201
+
+            app.celery.tasks.process_job.apply_async.assert_called_once_with(
+                ([str(job_id)]),
+                queue="process-job"
+            )
+
+            resp_json = json.loads(response.get_data(as_text=True))
+
+            assert resp_json['data']['id'] == str(job_id)
+            assert resp_json['data']['service'] == str(sample_template.service.id)
+            assert resp_json['data']['template'] == sample_template.id
+            assert resp_json['data']['original_file_name'] == 'thisisatest.csv'
+
+
+def test_create_job_returns_400_if_missing_data(notify_api, sample_template, mocker):
+    with notify_api.test_request_context():
+        with notify_api.test_client() as client:
+            mocker.patch('app.celery.tasks.process_job.apply_async')
+            data = {
+            }
+            path = '/service/{}/job'.format(sample_template.service.id)
+            auth_header = create_authorization_header(
+                service_id=sample_template.service.id,
+                path=path,
+                method='POST',
+                request_body=json.dumps(data))
+            headers = [('Content-Type', 'application/json'), auth_header]
+            response = client.post(
+                path,
+                data=json.dumps(data),
+                headers=headers)
+
+            resp_json = json.loads(response.get_data(as_text=True))
+            assert response.status_code == 400
+
+            app.celery.tasks.process_job.apply_async.assert_not_called()
+            assert resp_json['result'] == 'error'
+            assert 'Missing data for required field.' in resp_json['message']['original_file_name']
+            assert 'Missing data for required field.' in resp_json['message']['file_name']
+            assert 'Missing data for required field.' in resp_json['message']['notification_count']
+            assert 'Missing data for required field.' in resp_json['message']['id']
+            assert 'Missing data for required field.' in resp_json['message']['bucket_name']
+
+
+def test_create_job_returns_404_if_missing_service(notify_api, sample_template, mocker):
+    with notify_api.test_request_context():
+        with notify_api.test_client() as client:
+            mocker.patch('app.celery.tasks.process_job.apply_async')
+            random_id = str(uuid.uuid4())
+            data = {}
+            path = '/service/{}/job'.format(random_id)
+            auth_header = create_authorization_header(
+                service_id=sample_template.service.id,
+                path=path,
+                method='POST',
+                request_body=json.dumps(data))
+            headers = [('Content-Type', 'application/json'), auth_header]
+            response = client.post(
+                path,
+                data=json.dumps(data),
+                headers=headers)
+
+            resp_json = json.loads(response.get_data(as_text=True))
+            assert response.status_code == 404
+
+            app.celery.tasks.process_job.apply_async.assert_not_called()
+            print(resp_json)
+            assert resp_json['result'] == 'error'
+            assert resp_json['message'] == 'Service {} not found'.format(random_id)
+
+
+def test_get_update_job(notify_api, sample_job):
+    assert sample_job.status == 'pending'
+
+    job_id = str(sample_job.id)
+    service_id = str(sample_job.service.id)
+
+    update_data = {
+        'status': 'in progress'
+    }
+
+    with notify_api.test_request_context():
+        with notify_api.test_client() as client:
+            path = '/service/{}/job/{}'.format(service_id, job_id)
+
+            auth_header = create_authorization_header(
+                service_id=service_id,
+                path=path,
+                method='POST',
+                request_body=json.dumps(update_data))
+
+            headers = [('Content-Type', 'application/json'), auth_header]
+
+            response = client.post(path, headers=headers, data=json.dumps(update_data))
+
+            resp_json = json.loads(response.get_data(as_text=True))
+            assert response.status_code == 200
+            assert resp_json['data']['status'] == 'in progress'
+
+
+def _setup_jobs(notify_db, notify_db_session, template, number_of_jobs=5):
+    for i in range(number_of_jobs):
+        print(i)
+        create_job(
+            notify_db,
+            notify_db_session,
+            service=template.service,
+            template=template)

tests/app/notifications/test_rest.py

@@ -904,6 +904,7 @@ def test_should_allow_valid_email_notification_for_job(notify_api, sample_job, s
                 data=json.dumps(data),
                 headers=[('Content-Type', 'application/json'), auth_header])
 
+            print(json.loads(response.data))
             notification_id = json.loads(response.data)['notification_id']
             app.celery.tasks.send_email.apply_async.assert_called_once_with(
                 (str(sample_job.service_id),

tests/app/test_csv.py

@@ -0,0 +1,15 @@
+from app.csv import get_mobile_numbers_from_csv
+from tests.app import load_example_csv
+
+
+def test_should_process_single_phone_number_file():
+    sms_file = load_example_csv('sms')
+    len(get_mobile_numbers_from_csv(sms_file)) == 1
+    assert get_mobile_numbers_from_csv(sms_file)[0] == '+441234123123'
+
+
+def test_should_process_multple_phone_number_file_in_order():
+    sms_file = load_example_csv('multiple_sms')
+    len(get_mobile_numbers_from_csv(sms_file)) == 10
+    assert get_mobile_numbers_from_csv(sms_file)[0] == '+441234123121'
+    assert get_mobile_numbers_from_csv(sms_file)[9] == '+441234123120'