darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-03-24 04:00:09 -04:00
Code Issues Packages Projects Releases Wiki Activity

Move job processing into celery

Browse Source 
- brings boto S3 into new AWS folder
- CSV processing utils method

Rejigs the jobs rest endpoint - removes some now unused endpoints,

Calls to the task with the job, job processing in task, delegating SMS calls to the sms task
This commit is contained in:
Martyn Inglis
2016-02-24 17:12:30 +00:00
parent 1667f82df1
commit b3884e2d6c
20 changed files with 453 additions and 527 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
app/__init__.py
View File

@@ -1,3 +1,4 @@
import uuid
import os
import re
from flask import request, url_for
@@ -50,7 +51,7 @@ def create_app():
application.register_blueprint(user_blueprint, url_prefix='/user')
application.register_blueprint(template_blueprint)
application.register_blueprint(status_blueprint, url_prefix='/status')
application.register_blueprint(notifications_blueprint, url_prefix='/notifications')
application.register_blueprint(notifications_blueprint)
application.register_blueprint(job_blueprint)
return application
@@ -99,3 +100,7 @@ def email_safe(string):
character.lower() if character.isalnum() or character == "." else ""
for character in re.sub("\s+", ".", string.strip())
])
def create_uuid():
return str(uuid.uuid4())

7
app/aws/s3.py Normal file
View File

@@ -0,0 +1,7 @@
from boto3 import resource
def get_job_from_s3(bucket_name, job_id):
s3 = resource('s3')
key = s3.Object(bucket_name, '{}.csv'.format(job_id))
return key.get()['Body'].read().decode('utf-8')

31
app/celery/tasks.py
View File

@@ -1,11 +1,42 @@
from app import create_uuid
from app import notify_celery, encryption, firetext_client, aws_ses_client
from app.clients.email.aws_ses import AwsSesClientException
from app.clients.sms.firetext import FiretextClientException
from app.dao.templates_dao import dao_get_template_by_id
from app.dao.notifications_dao import save_notification
from app.dao.jobs_dao import dao_update_job, dao_get_job_by_id
from app.models import Notification
from flask import current_app
from sqlalchemy.exc import SQLAlchemyError
from app.aws import s3
from app.csv import get_mobile_numbers_from_csv
@notify_celery.task(name="process-job")
def process_job(job_id):
job = dao_get_job_by_id(job_id)
job.status = 'in progress'
dao_update_job(job)
file = s3.get_job_from_s3(job.bucket_name, job_id)
mobile_numbers = get_mobile_numbers_from_csv(file)
for mobile_number in mobile_numbers:
notification = encryption.encrypt({
'template': job.template_id,
'job': str(job.id),
'to': mobile_number
})
send_sms.apply_async((
str(job.service_id),
str(create_uuid()),
notification),
queue='sms'
)
job.status = 'finished'
dao_update_job(job)
@notify_celery.task(name="send-sms")

12
app/csv.py Normal file
View File

@@ -0,0 +1,12 @@
import csv
def get_mobile_numbers_from_csv(file_data):
numbers = []
reader = csv.DictReader(
file_data.splitlines(),
lineterminator='\n',
quoting=csv.QUOTE_NONE)
for i, row in enumerate(reader):
numbers.append(row['phone'].replace(' ', ''))
return numbers

29
app/dao/jobs_dao.py
View File

@@ -2,24 +2,23 @@ from app import db
from app.models import Job
def save_job(job, update_dict={}):
if update_dict:
update_dict.pop('id', None)
update_dict.pop('service', None)
update_dict.pop('template', None)
Job.query.filter_by(id=job.id).update(update_dict)
else:
db.session.add(job)
db.session.commit()
def get_job(service_id, job_id):
def dao_get_job_by_service_id_and_job_id(service_id, job_id):
return Job.query.filter_by(service_id=service_id, id=job_id).first()
def get_jobs_by_service(service_id):
def dao_get_jobs_by_service_id(service_id):
return Job.query.filter_by(service_id=service_id).all()
def _get_jobs():
return Job.query.all()
def dao_get_job_by_id(job_id):
return Job.query.filter_by(id=job_id).first()
def dao_create_job(job):
db.session.add(job)
db.session.commit()
def dao_update_job(job):
db.session.add(job)
db.session.commit()

2
app/errors.py
View File

@@ -43,7 +43,7 @@ def register_errors(blueprint):
@blueprint.app_errorhandler(DataError)
def no_result_found(e):
current_app.logger.error(e)
return jsonify(error="No result found"), 404
return jsonify(result="error", message="No result found"), 404
@blueprint.app_errorhandler(SQLAlchemyError)
def db_error(e):

159
app/job/rest.py
View File

@@ -1,148 +1,81 @@
import boto3
import json
from flask import (
Blueprint,
jsonify,
request,
current_app
request
)
from sqlalchemy.exc import DataError
from sqlalchemy.orm.exc import NoResultFound
from app.dao.jobs_dao import (
save_job,
get_job,
get_jobs_by_service
dao_create_job,
dao_get_job_by_service_id_and_job_id,
dao_get_jobs_by_service_id,
dao_update_job
)
from app.dao import notifications_dao
from app.dao.services_dao import (
dao_fetch_service_by_id
)
from app.schemas import (
job_schema,
jobs_schema,
job_schema_load_json,
notification_status_schema,
notifications_status_schema,
notification_status_schema_load_json
jobs_schema
)
from app.celery.tasks import process_job
job = Blueprint('job', __name__, url_prefix='/service/<service_id>/job')
from app.errors import register_errors
register_errors(job)
@job.route('/<job_id>', methods=['GET'])
def get_job_by_service_and_job_id(service_id, job_id):
job = dao_get_job_by_service_id_and_job_id(service_id, job_id)
if not job:
return jsonify(result="error", message="Job {} not found for service {}".format(job_id, service_id)), 404
data, errors = job_schema.dump(job)
return jsonify(data=data)
@job.route('', methods=['GET'])
def get_job_for_service(service_id, job_id=None):
if job_id:
try:
job = get_job(service_id, job_id)
if not job:
return jsonify(result="error", message="Job not found"), 404
data, errors = job_schema.dump(job)
return jsonify(data=data)
except DataError:
return jsonify(result="error", message="Invalid job id"), 400
else:
jobs = get_jobs_by_service(service_id)
data, errors = jobs_schema.dump(jobs)
return jsonify(data=data)
def get_jobs_by_service(service_id):
jobs = dao_get_jobs_by_service_id(service_id)
data, errors = jobs_schema.dump(jobs)
return jsonify(data=data)
@job.route('', methods=['POST'])
def create_job(service_id):
job, errors = job_schema.load(request.get_json())
service = dao_fetch_service_by_id(service_id)
if not service:
return jsonify(result="error", message="Service {} not found".format(service_id)), 404
data = request.get_json()
data.update({
"service": service_id
})
job, errors = job_schema.load(data)
if errors:
return jsonify(result="error", message=errors), 400
save_job(job)
_enqueue_job(job)
dao_create_job(job)
process_job.apply_async([str(job.id)], queue="process-job")
return jsonify(data=job_schema.dump(job).data), 201
@job.route('/<job_id>', methods=['PUT'])
@job.route('/<job_id>', methods=['POST'])
def update_job(service_id, job_id):
fetched_job = dao_get_job_by_service_id_and_job_id(service_id, job_id)
if not fetched_job:
return jsonify(result="error", message="Job {} not found for service {}".format(job_id, service_id)), 404
job = get_job(service_id, job_id)
update_dict, errors = job_schema_load_json.load(request.get_json())
current_data = dict(job_schema.dump(fetched_job).data.items())
current_data.update(request.get_json())
update_dict, errors = job_schema.load(current_data)
if errors:
return jsonify(result="error", message=errors), 400
save_job(job, update_dict=update_dict)
return jsonify(data=job_schema.dump(job).data), 200
@job.route('/<job_id>/notification', methods=['POST'])
def create_notification_for_job(service_id, job_id):
# TODO assert service_id == payload service id
# and same for job id
notification, errors = notification_status_schema.load(request.get_json())
if errors:
return jsonify(result="error", message=errors), 400
notifications_dao.save_notification(notification)
return jsonify(data=notification_status_schema.dump(notification).data), 201
@job.route('/<job_id>/notification', methods=['GET'])
@job.route('/<job_id>/notification/<notification_id>')
def get_notification_for_job(service_id, job_id, notification_id=None):
if notification_id:
try:
notification = notifications_dao.get_notification_for_job(service_id, job_id, notification_id)
data, errors = notification_status_schema.dump(notification)
return jsonify(data=data)
except DataError:
return jsonify(result="error", message="Invalid notification id"), 400
except NoResultFound:
return jsonify(result="error", message="Notification not found"), 404
else:
notifications = notifications_dao.get_notifications_for_job(service_id, job_id)
data, errors = notifications_status_schema.dump(notifications)
return jsonify(data=data)
@job.route('/<job_id>/notification/<notification_id>', methods=['PUT'])
def update_notification_for_job(service_id, job_id, notification_id):
notification = notifications_dao.get_notification_for_job(service_id, job_id, notification_id)
update_dict, errors = notification_status_schema_load_json.load(request.get_json())
if errors:
return jsonify(result="error", message=errors), 400
notifications_dao.save_notification(notification, update_dict=update_dict)
return jsonify(data=job_schema.dump(notification).data), 200
def _enqueue_job(job):
aws_region = current_app.config['AWS_REGION']
queue_name = current_app.config['NOTIFY_JOB_QUEUE']
queue = boto3.resource('sqs', region_name=aws_region).create_queue(QueueName=queue_name)
data = {
'id': str(job.id),
'service': str(job.service.id),
'template': job.template.id,
'bucket_name': job.bucket_name,
'file_name': job.file_name,
'original_file_name': job.original_file_name
}
job_json = json.dumps(data)
queue.send_message(MessageBody=job_json,
MessageAttributes={'id': {'StringValue': str(job.id), 'DataType': 'String'},
'service': {'StringValue': str(job.service.id), 'DataType': 'String'},
'template': {'StringValue': str(job.template.id), 'DataType': 'String'},
'bucket_name': {'StringValue': job.bucket_name, 'DataType': 'String'},
'file_name': {'StringValue': job.file_name, 'DataType': 'String'},
'original_file_name': {'StringValue': job.original_file_name,
'DataType': 'String'}})
dao_update_job(update_dict)
return jsonify(data=job_schema.dump(update_dict).data), 200

22
app/notifications/rest.py
View File

@@ -1,5 +1,3 @@
import uuid
from flask import (
Blueprint,
jsonify,
@@ -7,7 +5,7 @@ from flask import (
current_app
)
from app import api_user, encryption
from app import api_user, encryption, create_uuid
from app.dao import (
templates_dao,
services_dao,
@@ -34,11 +32,7 @@ SMS_NOTIFICATION = 'sms'
EMAIL_NOTIFICATION = 'email'
def create_notification_id():
return str(uuid.uuid4())
@notifications.route('/<string:notification_id>', methods=['GET'])
@notifications.route('/notifications/<string:notification_id>', methods=['GET'])
def get_notifications(notification_id):
try:
notification = notifications_dao.get_notification(api_user['client'], notification_id)
@@ -47,22 +41,22 @@ def get_notifications(notification_id):
return jsonify(result="error", message="not found"), 404
@notifications.route('/sms', methods=['POST'])
@notifications.route('/notifications/sms', methods=['POST'])
def create_sms_notification():
return send_notification(notification_type=SMS_NOTIFICATION, expects_job=False)
@notifications.route('/sms/service/<service_id>', methods=['POST'])
@notifications.route('/notifications/sms/service/<service_id>', methods=['POST'])
def create_sms_for_job(service_id):
return send_notification(service_id=service_id, notification_type=SMS_NOTIFICATION, expects_job=True)
@notifications.route('/email', methods=['POST'])
@notifications.route('/notifications/email', methods=['POST'])
def create_email_notification():
return send_notification(notification_type=EMAIL_NOTIFICATION, expects_job=False)
@notifications.route('/email/service/<service_id>', methods=['POST'])
@notifications.route('/notifications/email/service/<service_id>', methods=['POST'])
def create_email_notification_for_job(service_id):
return send_notification(service_id=service_id, notification_type=EMAIL_NOTIFICATION, expects_job=True)
@@ -98,7 +92,7 @@ def send_notification(notification_type, service_id=None, expects_job=False):
), 400
if expects_job:
job = jobs_dao.get_job(service_id, notification['job'])
job = jobs_dao.dao_get_job_by_service_id_and_job_id(service_id, notification['job'])
if not job:
return jsonify(result="error", message={'job': ['Job {} not found'.format(notification['job'])]}), 400
@@ -115,7 +109,7 @@ def send_notification(notification_type, service_id=None, expects_job=False):
return jsonify(
result="error", message={'to': ['Email address not permitted for restricted service']}), 400
notification_id = create_notification_id()
notification_id = create_uuid()
if notification_type is SMS_NOTIFICATION:
send_sms.apply_async((

2
scripts/run_celery.sh
View File

@@ -3,4 +3,4 @@
set -e
source environment.sh
celery -A run_celery.notify_celery worker --loglevel=INFO --logfile=/var/log/notify/application.log --concurrency=4 -Q sms,sms-code,email-code,email
celery -A run_celery.notify_celery worker --loglevel=INFO --logfile=/var/log/notify/application.log --concurrency=4 -Q sms,sms-code,email-code,email,process-job

11
test_csv_files/multiple_sms.csv Normal file
View File

@@ -0,0 +1,11 @@
phone
+441234123121
+441234123122
+441234123123
+441234123124
+441234123125
+441234123126
+441234123127
+441234123128
+441234123129
+441234123120
1 phone
2 +441234123121
3 +441234123122
4 +441234123123
5 +441234123124
6 +441234123125
7 +441234123126
8 +441234123127
9 +441234123128
10 +441234123129
11 +441234123120

2
test_csv_files/sms.csv Normal file
View File

@@ -0,0 +1,2 @@
phone
+441234123123
1 phone
2 +441234123123

7
tests/app/__init__.py
View File

@@ -0,0 +1,7 @@
import os
def load_example_csv(file):
file_path = os.path.join("test_csv_files", "{}.csv".format(file))
with open(file_path) as f:
return f.read()

50
tests/app/celery/test_tasks.py
View File

@@ -1,13 +1,49 @@
import uuid
import pytest
from flask import current_app
from app.celery.tasks import (send_sms, send_sms_code, send_email_code, send_email)
from app.celery.tasks import (send_sms, send_sms_code, send_email_code, send_email, process_job)
from app import (firetext_client, aws_ses_client, encryption)
from app.clients.email.aws_ses import AwsSesClientException
from app.clients.sms.firetext import FiretextClientException
from app.dao import notifications_dao
from app.dao import notifications_dao, jobs_dao
from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy.orm.exc import NoResultFound
from app.celery.tasks import s3
from app.celery import tasks
from tests.app import load_example_csv
def test_should_process_sms_job(sample_job, mocker):
mocker.patch('app.celery.tasks.s3.get_job_from_s3', return_value=load_example_csv('sms'))
mocker.patch('app.celery.tasks.send_sms.apply_async')
mocker.patch('app.encryption.encrypt', return_value="something_encrypted")
mocker.patch('app.celery.tasks.create_uuid', return_value="uuid")
process_job(sample_job.id)
s3.get_job_from_s3.assert_called_once_with(sample_job.bucket_name, sample_job.id)
tasks.send_sms.apply_async.assert_called_once_with(
(str(sample_job.service_id),
"uuid",
"something_encrypted"),
queue="sms"
)
job = jobs_dao.dao_get_job_by_id(sample_job.id)
assert job.status == 'finished'
def test_should_process_all_sms_job(sample_job, mocker):
mocker.patch('app.celery.tasks.s3.get_job_from_s3', return_value=load_example_csv('multiple_sms'))
mocker.patch('app.celery.tasks.send_sms.apply_async')
mocker.patch('app.encryption.encrypt', return_value="something_encrypted")
mocker.patch('app.celery.tasks.create_uuid', return_value="uuid")
process_job(sample_job.id)
s3.get_job_from_s3.assert_called_once_with(sample_job.bucket_name, sample_job.id)
tasks.send_sms.apply_async.call_count == 10
job = jobs_dao.dao_get_job_by_id(sample_job.id)
assert job.status == 'finished'
def test_should_send_template_to_correct_sms_provider_and_persist(sample_template, mocker):
@@ -218,7 +254,9 @@ def test_should_send_email_code(mocker):
send_email_code(encrypted_verification)
aws_ses_client.send_email.assert_called_once_with(current_app.config['VERIFY_CODE_FROM_EMAIL_ADDRESS'],
verification['to'],
"Verification code",
verification['secret_code'])
aws_ses_client.send_email.assert_called_once_with(
current_app.config['VERIFY_CODE_FROM_EMAIL_ADDRESS'],
verification['to'],
"Verification code",
verification['secret_code']
)

4
tests/app/conftest.py
View File

@@ -6,7 +6,7 @@ from app.dao.users_dao import (save_model_user, create_user_code, create_secret_
from app.dao.services_dao import dao_create_service
from app.dao.templates_dao import dao_create_template
from app.dao.api_key_dao import save_model_api_key
from app.dao.jobs_dao import save_job
from app.dao.jobs_dao import dao_create_job
from app.dao.notifications_dao import save_notification
import uuid
@@ -204,7 +204,7 @@ def sample_job(notify_db,
'notification_count': 1
}
job = Job(**data)
save_job(job)
dao_create_job(job)
return job

63
tests/app/dao/test_jobs_dao.py
View File

@@ -1,18 +1,16 @@
import uuid
import json
from app.dao.jobs_dao import (
save_job,
get_job,
get_jobs_by_service,
_get_jobs
dao_get_job_by_service_id_and_job_id,
dao_create_job,
dao_update_job,
dao_get_jobs_by_service_id
)
from app.models import Job
def test_save_job(notify_db, notify_db_session, sample_template):
def test_create_job(sample_template):
assert Job.query.count() == 0
job_id = uuid.uuid4()
@@ -29,39 +27,33 @@ def test_save_job(notify_db, notify_db_session, sample_template):
}
job = Job(**data)
save_job(job)
dao_create_job(job)
assert Job.query.count() == 1
job_from_db = Job.query.get(job_id)
assert job == job_from_db
def test_get_job_by_id(notify_db, notify_db_session, sample_job):
job_from_db = get_job(sample_job.service.id, sample_job.id)
def test_get_job_by_id(sample_job):
job_from_db = dao_get_job_by_service_id_and_job_id(sample_job.service.id, sample_job.id)
assert sample_job == job_from_db
def test_get_jobs_for_service(notify_db, notify_db_session, sample_template):
from tests.app.conftest import sample_job as create_job
from tests.app.conftest import sample_service as create_service
from tests.app.conftest import sample_template as create_template
from tests.app.conftest import sample_user as create_user
one_job = create_job(notify_db, notify_db_session, sample_template.service,
sample_template)
one_job = create_job(notify_db, notify_db_session, sample_template.service, sample_template)
other_user = create_user(notify_db, notify_db_session,
email="test@digital.cabinet-office.gov.uk")
other_service = create_service(notify_db, notify_db_session,
user=other_user, service_name="other service")
other_template = create_template(notify_db, notify_db_session,
service=other_service)
other_job = create_job(notify_db, notify_db_session, service=other_service,
template=other_template)
other_user = create_user(notify_db, notify_db_session, email="test@digital.cabinet-office.gov.uk")
other_service = create_service(notify_db, notify_db_session, user=other_user, service_name="other service")
other_template = create_template(notify_db, notify_db_session, service=other_service)
other_job = create_job(notify_db, notify_db_session, service=other_service, template=other_template)
one_job_from_db = get_jobs_by_service(one_job.service_id)
other_job_from_db = get_jobs_by_service(other_job.service_id)
one_job_from_db = dao_get_jobs_by_service_id(one_job.service_id)
other_job_from_db = dao_get_jobs_by_service_id(other_job.service_id)
assert len(one_job_from_db) == 1
assert one_job == one_job_from_db[0]
@@ -72,31 +64,12 @@ def test_get_jobs_for_service(notify_db, notify_db_session, sample_template):
assert one_job_from_db != other_job_from_db
def test_get_all_jobs(notify_db, notify_db_session, sample_template):
from tests.app.conftest import sample_job as create_job
for i in range(5):
create_job(notify_db,
notify_db_session,
sample_template.service,
sample_template)
jobs_from_db = _get_jobs()
assert len(jobs_from_db) == 5
def test_update_job(notify_db, notify_db_session, sample_job):
def test_update_job(sample_job):
assert sample_job.status == 'pending'
update_dict = {
'id': sample_job.id,
'service': sample_job.service.id,
'template': sample_job.template.id,
'bucket_name': sample_job.bucket_name,
'file_name': sample_job.file_name,
'original_file_name': sample_job.original_file_name,
'status': 'in progress'
}
sample_job.status = 'in progress'
save_job(sample_job, update_dict=update_dict)
dao_update_job(sample_job)
job_from_db = Job.query.get(sample_job.id)

2
tests/app/dao/test_notification_dao.py
View File

@@ -8,7 +8,7 @@ from app.dao.notifications_dao import (
)
def test_save_notification(notify_db, notify_db_session, sample_template, sample_job):
def test_save_notification(sample_template, sample_job):
assert Notification.query.count() == 0
to = '+44709123456'

328
tests/app/job/test_job_rest.py
View File

@@ -1,328 +0,0 @@
import boto3
import moto
import json
import uuid
from flask import url_for
from tests import create_authorization_header
from tests.app.conftest import sample_job as create_job
def test_get_jobs(notify_api, notify_db, notify_db_session, sample_template):
_setup_jobs(notify_db, notify_db_session, sample_template)
service_id = sample_template.service.id
with notify_api.test_request_context():
with notify_api.test_client() as client:
path = url_for('job.get_job_for_service', service_id=service_id)
auth_header = create_authorization_header(service_id=service_id,
path=path,
method='GET')
response = client.get(path, headers=[auth_header])
assert response.status_code == 200
resp_json = json.loads(response.get_data(as_text=True))
assert len(resp_json['data']) == 5
def test_get_job_with_invalid_id_returns400(notify_api, notify_db,
notify_db_session,
sample_template):
service_id = sample_template.service.id
with notify_api.test_request_context():
with notify_api.test_client() as client:
path = url_for('job.get_job_for_service', job_id='invalid_id', service_id=service_id)
auth_header = create_authorization_header(service_id=sample_template.service.id,
path=path,
method='GET')
response = client.get(path, headers=[auth_header])
assert response.status_code == 400
resp_json = json.loads(response.get_data(as_text=True))
assert resp_json == {'message': 'Invalid job id',
'result': 'error'}
def test_get_job_with_unknown_id_returns404(notify_api, notify_db,
notify_db_session,
sample_template):
random_id = str(uuid.uuid4())
service_id = sample_template.service.id
with notify_api.test_request_context():
with notify_api.test_client() as client:
path = url_for('job.get_job_for_service', job_id=random_id, service_id=service_id)
auth_header = create_authorization_header(service_id=sample_template.service.id,
path=path,
method='GET')
response = client.get(path, headers=[auth_header])
assert response.status_code == 404
resp_json = json.loads(response.get_data(as_text=True))
assert resp_json == {'message': 'Job not found', 'result': 'error'}
def test_get_job_by_id(notify_api, notify_db, notify_db_session,
sample_job):
job_id = str(sample_job.id)
service_id = sample_job.service.id
with notify_api.test_request_context():
with notify_api.test_client() as client:
path = url_for('job.get_job_for_service', job_id=job_id, service_id=service_id)
auth_header = create_authorization_header(service_id=sample_job.service.id,
path=path,
method='GET')
response = client.get(path, headers=[auth_header])
assert response.status_code == 200
resp_json = json.loads(response.get_data(as_text=True))
assert resp_json['data']['id'] == job_id
@moto.mock_sqs
def test_create_job(notify_api, notify_db, notify_db_session, sample_template):
job_id = uuid.uuid4()
template_id = sample_template.id
service_id = sample_template.service.id
original_file_name = 'thisisatest.csv'
bucket_name = 'service-{}-notify'.format(service_id)
file_name = '{}.csv'.format(job_id)
data = {
'id': str(job_id),
'service': str(service_id),
'template': template_id,
'original_file_name': original_file_name,
'bucket_name': bucket_name,
'file_name': file_name,
'notification_count': 1
}
with notify_api.test_request_context():
with notify_api.test_client() as client:
path = url_for('job.create_job', service_id=service_id)
auth_header = create_authorization_header(service_id=sample_template.service.id,
path=path,
method='POST',
request_body=json.dumps(data))
headers = [('Content-Type', 'application/json'), auth_header]
response = client.post(
path,
data=json.dumps(data),
headers=headers)
assert response.status_code == 201
resp_json = json.loads(response.get_data(as_text=True))
assert resp_json['data']['id'] == str(job_id)
assert resp_json['data']['service'] == str(service_id)
assert resp_json['data']['template'] == template_id
assert resp_json['data']['original_file_name'] == original_file_name
boto3.setup_default_session(region_name='eu-west-1')
q = boto3.resource('sqs').get_queue_by_name(QueueName=notify_api.config['NOTIFY_JOB_QUEUE'])
messages = q.receive_messages()
assert len(messages) == 1
expected_message = json.loads(messages[0].body)
assert expected_message['id'] == str(job_id)
assert expected_message['service'] == str(service_id)
assert expected_message['template'] == template_id
assert expected_message['bucket_name'] == bucket_name
def test_get_update_job_status(notify_api,
notify_db,
notify_db_session,
sample_job):
assert sample_job.status == 'pending'
job_id = str(sample_job.id)
service_id = str(sample_job.service.id)
update_data = {
'id': job_id,
'service': service_id,
'template': sample_job.template.id,
'bucket_name': sample_job.bucket_name,
'file_name': sample_job.file_name,
'original_file_name': sample_job.original_file_name,
'status': 'in progress',
'notification_count': 1
}
with notify_api.test_request_context():
with notify_api.test_client() as client:
path = url_for('job.update_job', service_id=service_id, job_id=job_id)
auth_header = create_authorization_header(service_id=service_id,
path=path,
method='PUT',
request_body=json.dumps(update_data))
headers = [('Content-Type', 'application/json'), auth_header]
response = client.put(path, headers=headers, data=json.dumps(update_data))
assert response.status_code == 200
resp_json = json.loads(response.get_data(as_text=True))
assert resp_json['data']['status'] == 'in progress'
def test_get_notification(notify_api, notify_db, notify_db_session, sample_notification):
with notify_api.test_request_context():
with notify_api.test_client() as client:
path = url_for('job.get_notification_for_job',
service_id=sample_notification.service.id,
job_id=sample_notification.job.id,
notification_id=sample_notification.id)
auth_header = create_authorization_header(service_id=sample_notification.service.id,
path=path,
method='GET')
headers = [('Content-Type', 'application/json'), auth_header]
response = client.get(path, headers=headers)
resp_json = json.loads(response.get_data(as_text=True))
assert str(sample_notification.id) == resp_json['data']['id']
assert str(sample_notification.service.id) == resp_json['data']['service']
assert sample_notification.template.id == resp_json['data']['template']
assert str(sample_notification.job.id) == resp_json['data']['job']
assert sample_notification.status == resp_json['data']['status']
def test_get_notifications(notify_api, notify_db, notify_db_session, sample_job):
from tests.app.conftest import sample_notification
for i in range(0, 5):
sample_notification(notify_db,
notify_db_session,
service=sample_job.service,
template=sample_job.template,
job=sample_job)
service_id = str(sample_job.service.id)
job_id = str(sample_job.id)
with notify_api.test_request_context():
with notify_api.test_client() as client:
path = url_for('job.get_notification_for_job',
service_id=service_id,
job_id=job_id)
auth_header = create_authorization_header(service_id=service_id,
path=path,
method='GET')
headers = [('Content-Type', 'application/json'), auth_header]
response = client.get(path, headers=headers)
resp_json = json.loads(response.get_data(as_text=True))
assert len(resp_json['data']) == 5
def test_add_notification(notify_api, notify_db, notify_db_session, sample_job):
to = '+44709123456'
data = {
'to': to,
'job': str(sample_job.id),
'service': str(sample_job.service.id),
'template': sample_job.template.id
}
with notify_api.test_request_context():
with notify_api.test_client() as client:
path = url_for('job.create_notification_for_job',
service_id=sample_job.service.id,
job_id=sample_job.id)
auth_header = create_authorization_header(service_id=sample_job.service.id,
path=path,
method='POST',
request_body=json.dumps(data))
headers = [('Content-Type', 'application/json'), auth_header]
response = client.post(path, headers=headers, data=json.dumps(data))
resp_json = json.loads(response.get_data(as_text=True))
assert resp_json['data']['id']
assert data['to'] == resp_json['data']['to']
assert data['service'] == resp_json['data']['service']
assert data['template'] == resp_json['data']['template']
assert data['job'] == resp_json['data']['job']
assert 'sent' == resp_json['data']['status']
def test_add_notification_with_id(notify_api, notify_db, notify_db_session, sample_job):
notification_id = str(uuid.uuid4())
to = '+44709123456'
data = {
'id': notification_id,
'to': to,
'job': str(sample_job.id),
'service': str(sample_job.service.id),
'template': sample_job.template.id
}
with notify_api.test_request_context():
with notify_api.test_client() as client:
path = url_for('job.create_notification_for_job',
service_id=sample_job.service.id,
job_id=sample_job.id)
auth_header = create_authorization_header(service_id=sample_job.service.id,
path=path,
method='POST',
request_body=json.dumps(data))
headers = [('Content-Type', 'application/json'), auth_header]
response = client.post(path, headers=headers, data=json.dumps(data))
resp_json = json.loads(response.get_data(as_text=True))
assert resp_json['data']['id'] == notification_id
assert data['to'] == resp_json['data']['to']
assert data['service'] == resp_json['data']['service']
assert data['template'] == resp_json['data']['template']
assert data['job'] == resp_json['data']['job']
assert 'sent' == resp_json['data']['status']
def test_update_notification(notify_api, notify_db, notify_db_session, sample_notification):
assert sample_notification.status == 'sent'
update_data = {
'id': str(sample_notification.id),
'to': sample_notification.to,
'job': str(sample_notification.job.id),
'service': str(sample_notification.service.id),
'template': sample_notification.template.id,
'status': 'failed'
}
with notify_api.test_request_context():
with notify_api.test_client() as client:
path = url_for('job.update_notification_for_job',
service_id=sample_notification.service.id,
job_id=sample_notification.job.id,
notification_id=sample_notification.id)
auth_header = create_authorization_header(service_id=sample_notification.service.id,
path=path,
method='PUT',
request_body=json.dumps(update_data))
headers = [('Content-Type', 'application/json'), auth_header]
response = client.put(path, headers=headers, data=json.dumps(update_data))
resp_json = json.loads(response.get_data(as_text=True))
assert update_data['id'] == resp_json['data']['id']
assert 'failed' == resp_json['data']['status']
def _setup_jobs(notify_db, notify_db_session, template, number_of_jobs=5):
for i in range(number_of_jobs):
create_job(notify_db, notify_db_session, service=template.service,
template=template)

226
tests/app/job/test_rest.py Normal file
View File

@@ -0,0 +1,226 @@
import json
import uuid
import app.celery.tasks
from tests import create_authorization_header
from tests.app.conftest import sample_job as create_job
def test_get_jobs(notify_api, notify_db, notify_db_session, sample_template):
_setup_jobs(notify_db, notify_db_session, sample_template)
service_id = sample_template.service.id
with notify_api.test_request_context():
with notify_api.test_client() as client:
path = '/service/{}/job'.format(service_id)
auth_header = create_authorization_header(
service_id=service_id,
path=path,
method='GET')
response = client.get(path, headers=[auth_header])
assert response.status_code == 200
resp_json = json.loads(response.get_data(as_text=True))
assert len(resp_json['data']) == 5
def test_get_job_with_invalid_service_id_returns404(notify_api, sample_api_key, sample_service):
with notify_api.test_request_context():
with notify_api.test_client() as client:
path = '/service/{}/job'.format(sample_service.id)
auth_header = create_authorization_header(
service_id=sample_service.id,
path=path,
method='GET')
response = client.get(path, headers=[auth_header])
assert response.status_code == 200
resp_json = json.loads(response.get_data(as_text=True))
assert len(resp_json['data']) == 0
def test_get_job_with_invalid_job_id_returns404(notify_api, sample_template):
service_id = sample_template.service.id
with notify_api.test_request_context():
with notify_api.test_client() as client:
path = '/service/{}/job/{}'.format(service_id, "bad-id")
auth_header = create_authorization_header(
service_id=sample_template.service.id,
path=path,
method='GET')
response = client.get(path, headers=[auth_header])
assert response.status_code == 404
resp_json = json.loads(response.get_data(as_text=True))
print(resp_json)
assert resp_json['result'] == 'error'
assert resp_json['message'] == 'No result found'
def test_get_job_with_unknown_id_returns404(notify_api, sample_template):
random_id = str(uuid.uuid4())
service_id = sample_template.service.id
with notify_api.test_request_context():
with notify_api.test_client() as client:
path = '/service/{}/job/{}'.format(service_id, random_id)
auth_header = create_authorization_header(
service_id=sample_template.service.id,
path=path,
method='GET')
response = client.get(path, headers=[auth_header])
assert response.status_code == 404
resp_json = json.loads(response.get_data(as_text=True))
assert resp_json == {
'message': 'Job {} not found for service {}'.format(random_id, service_id),
'result': 'error'
}
def test_get_job_by_id(notify_api, sample_job):
job_id = str(sample_job.id)
service_id = sample_job.service.id
with notify_api.test_request_context():
with notify_api.test_client() as client:
path = '/service/{}/job/{}'.format(service_id, job_id)
auth_header = create_authorization_header(
service_id=sample_job.service.id,
path=path,
method='GET')
response = client.get(path, headers=[auth_header])
assert response.status_code == 200
resp_json = json.loads(response.get_data(as_text=True))
assert resp_json['data']['id'] == job_id
def test_create_job(notify_api, sample_template, mocker):
with notify_api.test_request_context():
with notify_api.test_client() as client:
mocker.patch('app.celery.tasks.process_job.apply_async')
job_id = uuid.uuid4()
data = {
'id': str(job_id),
'service': str(sample_template.service.id),
'template': sample_template.id,
'original_file_name': 'thisisatest.csv',
'bucket_name': 'service-{}-notify'.format(sample_template.service.id),
'file_name': '{}.csv'.format(job_id),
'notification_count': 1
}
path = '/service/{}/job'.format(sample_template.service.id)
auth_header = create_authorization_header(
service_id=sample_template.service.id,
path=path,
method='POST',
request_body=json.dumps(data))
headers = [('Content-Type', 'application/json'), auth_header]
response = client.post(
path,
data=json.dumps(data),
headers=headers)
assert response.status_code == 201
app.celery.tasks.process_job.apply_async.assert_called_once_with(
([str(job_id)]),
queue="process-job"
)
resp_json = json.loads(response.get_data(as_text=True))
assert resp_json['data']['id'] == str(job_id)
assert resp_json['data']['service'] == str(sample_template.service.id)
assert resp_json['data']['template'] == sample_template.id
assert resp_json['data']['original_file_name'] == 'thisisatest.csv'
def test_create_job_returns_400_if_missing_data(notify_api, sample_template, mocker):
with notify_api.test_request_context():
with notify_api.test_client() as client:
mocker.patch('app.celery.tasks.process_job.apply_async')
data = {
}
path = '/service/{}/job'.format(sample_template.service.id)
auth_header = create_authorization_header(
service_id=sample_template.service.id,
path=path,
method='POST',
request_body=json.dumps(data))
headers = [('Content-Type', 'application/json'), auth_header]
response = client.post(
path,
data=json.dumps(data),
headers=headers)
resp_json = json.loads(response.get_data(as_text=True))
assert response.status_code == 400
app.celery.tasks.process_job.apply_async.assert_not_called()
assert resp_json['result'] == 'error'
assert 'Missing data for required field.' in resp_json['message']['original_file_name']
assert 'Missing data for required field.' in resp_json['message']['file_name']
assert 'Missing data for required field.' in resp_json['message']['notification_count']
assert 'Missing data for required field.' in resp_json['message']['id']
assert 'Missing data for required field.' in resp_json['message']['bucket_name']
def test_create_job_returns_404_if_missing_service(notify_api, sample_template, mocker):
with notify_api.test_request_context():
with notify_api.test_client() as client:
mocker.patch('app.celery.tasks.process_job.apply_async')
random_id = str(uuid.uuid4())
data = {}
path = '/service/{}/job'.format(random_id)
auth_header = create_authorization_header(
service_id=sample_template.service.id,
path=path,
method='POST',
request_body=json.dumps(data))
headers = [('Content-Type', 'application/json'), auth_header]
response = client.post(
path,
data=json.dumps(data),
headers=headers)
resp_json = json.loads(response.get_data(as_text=True))
assert response.status_code == 404
app.celery.tasks.process_job.apply_async.assert_not_called()
print(resp_json)
assert resp_json['result'] == 'error'
assert resp_json['message'] == 'Service {} not found'.format(random_id)
def test_get_update_job(notify_api, sample_job):
assert sample_job.status == 'pending'
job_id = str(sample_job.id)
service_id = str(sample_job.service.id)
update_data = {
'status': 'in progress'
}
with notify_api.test_request_context():
with notify_api.test_client() as client:
path = '/service/{}/job/{}'.format(service_id, job_id)
auth_header = create_authorization_header(
service_id=service_id,
path=path,
method='POST',
request_body=json.dumps(update_data))
headers = [('Content-Type', 'application/json'), auth_header]
response = client.post(path, headers=headers, data=json.dumps(update_data))
resp_json = json.loads(response.get_data(as_text=True))
assert response.status_code == 200
assert resp_json['data']['status'] == 'in progress'
def _setup_jobs(notify_db, notify_db_session, template, number_of_jobs=5):
for i in range(number_of_jobs):
print(i)
create_job(
notify_db,
notify_db_session,
service=template.service,
template=template)

1
tests/app/notifications/test_rest.py
View File

@@ -904,6 +904,7 @@ def test_should_allow_valid_email_notification_for_job(notify_api, sample_job, s
data=json.dumps(data),
headers=[('Content-Type', 'application/json'), auth_header])
print(json.loads(response.data))
notification_id = json.loads(response.data)['notification_id']
app.celery.tasks.send_email.apply_async.assert_called_once_with(
(str(sample_job.service_id),

15
tests/app/test_csv.py Normal file
View File

@@ -0,0 +1,15 @@
from app.csv import get_mobile_numbers_from_csv
from tests.app import load_example_csv
def test_should_process_single_phone_number_file():
sms_file = load_example_csv('sms')
len(get_mobile_numbers_from_csv(sms_file)) == 1
assert get_mobile_numbers_from_csv(sms_file)[0] == '+441234123123'
def test_should_process_multple_phone_number_file_in_order():
sms_file = load_example_csv('multiple_sms')
len(get_mobile_numbers_from_csv(sms_file)) == 10
assert get_mobile_numbers_from_csv(sms_file)[0] == '+441234123121'
assert get_mobile_numbers_from_csv(sms_file)[9] == '+441234123120'