Removed the 'delete job' tasks from the tasks.py file

- this was called after processing a job.
- now to be called  on a schedule.

app/celery/tasks.py

@@ -101,20 +101,11 @@ def process_job(job_id):
     job.processing_started = start
     job.processing_finished = finished
     dao_update_job(job)
-    remove_job.apply_async((str(job_id),), queue='remove-job')
     current_app.logger.info(
         "Job {} created at {} started at {} finished at {}".format(job_id, job.created_at, start, finished)
     )
 
 
-@notify_celery.task(name="remove-job")
-@statsd(namespace="tasks")
-def remove_job(job_id):
-    job = dao_get_job_by_id(job_id)
-    s3.remove_job_from_s3(job.service.id, str(job_id))
-    current_app.logger.info("Job {} has been removed from s3.".format(job_id))
-
-
 @notify_celery.task(bind=True, name="send-sms", max_retries=5, default_retry_delay=300)
 @statsd(namespace="tasks")
 def send_sms(self,

tests/app/celery/test_tasks.py

@@ -10,7 +10,7 @@ from sqlalchemy.orm.exc import NoResultFound
 from app import (encryption, DATETIME_FORMAT)
 from app.celery import provider_tasks
 from app.celery import tasks
-from app.celery.tasks import s3, remove_job
+from app.celery.tasks import s3
 from app.celery.tasks import (
     send_sms,
     process_job,
@@ -55,13 +55,12 @@ def _notification_json(template, to, personalisation=None, job_id=None, row_numb
 
 def test_should_have_decorated_tasks_functions():
     assert process_job.__wrapped__.__name__ == 'process_job'
-    assert remove_job.__wrapped__.__name__ == 'remove_job'
     assert send_sms.__wrapped__.__name__ == 'send_sms'
     assert send_email.__wrapped__.__name__ == 'send_email'
 
 
 @freeze_time("2016-01-01 11:09:00.061258")
-def test_should_process_sms_job(sample_job, mocker, mock_celery_remove_job):
+def test_should_process_sms_job(sample_job, mocker):
     mocker.patch('app.celery.tasks.s3.get_job_from_s3', return_value=load_example_csv('sms'))
     mocker.patch('app.celery.tasks.send_sms.apply_async')
     mocker.patch('app.encryption.encrypt', return_value="something_encrypted")
@@ -91,8 +90,7 @@ def test_should_process_sms_job(sample_job, mocker, mock_celery_remove_job):
 @freeze_time("2016-01-01 11:09:00.061258")
 def test_should_not_process_sms_job_if_would_exceed_send_limits(notify_db,
                                                                 notify_db_session,
-                                                                mocker,
+                                                                mocker):
-                                                                mock_celery_remove_job):
     service = sample_service(notify_db, notify_db_session, limit=9)
     job = sample_job(notify_db, notify_db_session, service=service, notification_count=10)
 
@@ -107,13 +105,11 @@ def test_should_not_process_sms_job_if_would_exceed_send_limits(notify_db,
     job = jobs_dao.dao_get_job_by_id(job.id)
     assert job.status == 'sending limits exceeded'
     tasks.send_sms.apply_async.assert_not_called()
-    mock_celery_remove_job.assert_not_called()
 
 
 def test_should_not_process_sms_job_if_would_exceed_send_limits_inc_today(notify_db,
                                                                           notify_db_session,
-                                                                          mocker,
+                                                                          mocker):
-                                                                          mock_celery_remove_job):
     service = sample_service(notify_db, notify_db_session, limit=1)
     job = sample_job(notify_db, notify_db_session, service=service)
 
@@ -130,7 +126,6 @@ def test_should_not_process_sms_job_if_would_exceed_send_limits_inc_today(notify
     assert job.status == 'sending limits exceeded'
     s3.get_job_from_s3.assert_not_called()
     tasks.send_sms.apply_async.assert_not_called()
-    mock_celery_remove_job.assert_not_called()
 
 
 def test_should_not_process_email_job_if_would_exceed_send_limits_inc_today(notify_db, notify_db_session, mocker):
@@ -175,8 +170,7 @@ def test_should_not_process_email_job_if_would_exceed_send_limits(notify_db, not
 @freeze_time("2016-01-01 11:09:00.061258")
 def test_should_process_email_job_if_exactly_on_send_limits(notify_db,
                                                             notify_db_session,
-                                                            mocker,
+                                                            mocker):
-                                                            mock_celery_remove_job):
     service = sample_service(notify_db, notify_db_session, limit=10)
     template = sample_email_template(notify_db, notify_db_session, service=service)
     job = sample_job(notify_db, notify_db_session, service=service, template=template, notification_count=10)
@@ -203,10 +197,9 @@ def test_should_process_email_job_if_exactly_on_send_limits(notify_db,
         ),
         queue="db-email"
     )
-    mock_celery_remove_job.assert_called_once_with((str(job.id),), queue="remove-job")
 
 
-def test_should_not_create_send_task_for_empty_file(sample_job, mocker, mock_celery_remove_job):
+def test_should_not_create_send_task_for_empty_file(sample_job, mocker):
     mocker.patch('app.celery.tasks.s3.get_job_from_s3', return_value=load_example_csv('empty'))
     mocker.patch('app.celery.tasks.send_sms.apply_async')
 
@@ -218,11 +211,10 @@ def test_should_not_create_send_task_for_empty_file(sample_job, mocker, mock_cel
     )
     job = jobs_dao.dao_get_job_by_id(sample_job.id)
     assert job.status == 'finished'
-    tasks.send_sms.apply_async.assert_not_called
 
 
 @freeze_time("2016-01-01 11:09:00.061258")
-def test_should_process_email_job(sample_email_job, mocker, mock_celery_remove_job):
+def test_should_process_email_job(sample_email_job, mocker):
     mocker.patch('app.celery.tasks.s3.get_job_from_s3', return_value=load_example_csv('email'))
     mocker.patch('app.celery.tasks.send_email.apply_async')
     mocker.patch('app.encryption.encrypt', return_value="something_encrypted")
@@ -249,13 +241,11 @@ def test_should_process_email_job(sample_email_job, mocker, mock_celery_remove_j
     )
     job = jobs_dao.dao_get_job_by_id(sample_email_job.id)
     assert job.status == 'finished'
-    mock_celery_remove_job.assert_called_once_with((str(job.id),), queue="remove-job")
 
 
 def test_should_process_all_sms_job(sample_job,
                                     sample_job_with_placeholdered_template,
-                                    mocker,
+                                    mocker):
-                                    mock_celery_remove_job):
     mocker.patch('app.celery.tasks.s3.get_job_from_s3', return_value=load_example_csv('multiple_sms'))
     mocker.patch('app.celery.tasks.send_sms.apply_async')
     mocker.patch('app.encryption.encrypt', return_value="something_encrypted")

tests/app/conftest.py

@@ -479,11 +479,6 @@ def mock_encryption(mocker):
     return mocker.patch('app.encryption.encrypt', return_value="something_encrypted")
 
 
-@pytest.fixture(scope='function')
-def mock_celery_remove_job(mocker):
-    return mocker.patch('app.celery.tasks.remove_job.apply_async')
-
-
 @pytest.fixture(scope='function')
 def sample_invited_user(notify_db,
                         notify_db_session,