Move job processing into celery

- brings boto S3 into new AWS folder
- CSV processing utils method

Rejigs the jobs rest endpoint - removes some now unused endpoints,

Calls to the task with the job, job processing in task, delegating SMS calls to the sms task
This commit is contained in:
Martyn Inglis
2016-02-24 17:12:30 +00:00
parent 1667f82df1
commit b3884e2d6c
20 changed files with 453 additions and 527 deletions

View File

@@ -1,18 +1,16 @@
import uuid
import json
from app.dao.jobs_dao import (
save_job,
get_job,
get_jobs_by_service,
_get_jobs
dao_get_job_by_service_id_and_job_id,
dao_create_job,
dao_update_job,
dao_get_jobs_by_service_id
)
from app.models import Job
def test_save_job(notify_db, notify_db_session, sample_template):
def test_create_job(sample_template):
assert Job.query.count() == 0
job_id = uuid.uuid4()
@@ -29,39 +27,33 @@ def test_save_job(notify_db, notify_db_session, sample_template):
}
job = Job(**data)
save_job(job)
dao_create_job(job)
assert Job.query.count() == 1
job_from_db = Job.query.get(job_id)
assert job == job_from_db
def test_get_job_by_id(notify_db, notify_db_session, sample_job):
job_from_db = get_job(sample_job.service.id, sample_job.id)
def test_get_job_by_id(sample_job):
job_from_db = dao_get_job_by_service_id_and_job_id(sample_job.service.id, sample_job.id)
assert sample_job == job_from_db
def test_get_jobs_for_service(notify_db, notify_db_session, sample_template):
from tests.app.conftest import sample_job as create_job
from tests.app.conftest import sample_service as create_service
from tests.app.conftest import sample_template as create_template
from tests.app.conftest import sample_user as create_user
one_job = create_job(notify_db, notify_db_session, sample_template.service,
sample_template)
one_job = create_job(notify_db, notify_db_session, sample_template.service, sample_template)
other_user = create_user(notify_db, notify_db_session,
email="test@digital.cabinet-office.gov.uk")
other_service = create_service(notify_db, notify_db_session,
user=other_user, service_name="other service")
other_template = create_template(notify_db, notify_db_session,
service=other_service)
other_job = create_job(notify_db, notify_db_session, service=other_service,
template=other_template)
other_user = create_user(notify_db, notify_db_session, email="test@digital.cabinet-office.gov.uk")
other_service = create_service(notify_db, notify_db_session, user=other_user, service_name="other service")
other_template = create_template(notify_db, notify_db_session, service=other_service)
other_job = create_job(notify_db, notify_db_session, service=other_service, template=other_template)
one_job_from_db = get_jobs_by_service(one_job.service_id)
other_job_from_db = get_jobs_by_service(other_job.service_id)
one_job_from_db = dao_get_jobs_by_service_id(one_job.service_id)
other_job_from_db = dao_get_jobs_by_service_id(other_job.service_id)
assert len(one_job_from_db) == 1
assert one_job == one_job_from_db[0]
@@ -72,31 +64,12 @@ def test_get_jobs_for_service(notify_db, notify_db_session, sample_template):
assert one_job_from_db != other_job_from_db
def test_get_all_jobs(notify_db, notify_db_session, sample_template):
from tests.app.conftest import sample_job as create_job
for i in range(5):
create_job(notify_db,
notify_db_session,
sample_template.service,
sample_template)
jobs_from_db = _get_jobs()
assert len(jobs_from_db) == 5
def test_update_job(notify_db, notify_db_session, sample_job):
def test_update_job(sample_job):
assert sample_job.status == 'pending'
update_dict = {
'id': sample_job.id,
'service': sample_job.service.id,
'template': sample_job.template.id,
'bucket_name': sample_job.bucket_name,
'file_name': sample_job.file_name,
'original_file_name': sample_job.original_file_name,
'status': 'in progress'
}
sample_job.status = 'in progress'
save_job(sample_job, update_dict=update_dict)
dao_update_job(sample_job)
job_from_db = Job.query.get(sample_job.id)

View File

@@ -8,7 +8,7 @@ from app.dao.notifications_dao import (
)
def test_save_notification(notify_db, notify_db_session, sample_template, sample_job):
def test_save_notification(sample_template, sample_job):
assert Notification.query.count() == 0
to = '+44709123456'