Merge pull request #2709 from alphagov/delete-old-task

Remove task that no longer runs

app/aws/s3.py

@@ -97,13 +97,6 @@ def remove_s3_object(bucket_name, object_key):
     return obj.delete()
 
 
-def remove_transformed_dvla_file(job_id):
-    bucket_name = current_app.config['DVLA_BUCKETS']['job']
-    file_location = '{}-dvla-job.text'.format(job_id)
-    obj = get_s3_object(bucket_name, file_location)
-    return obj.delete()
-
-
 def get_list_of_files_by_suffix(bucket_name, subfolder='', suffix='', last_modified=None):
     s3_client = client('s3', current_app.config['AWS_REGION'])
     paginator = s3_client.get_paginator('list_objects_v2')

app/celery/nightly_tasks.py

@@ -214,16 +214,6 @@ def delete_inbound_sms():
         raise
 
 
-@notify_celery.task(name="remove_transformed_dvla_files")
-@cronitor("remove_transformed_dvla_files")
-@statsd(namespace="tasks")
-def remove_transformed_dvla_files():
-    jobs = dao_get_jobs_older_than_data_retention(notification_types=[LETTER_TYPE])
-    for job in jobs:
-        s3.remove_transformed_dvla_file(job.id)
-        current_app.logger.info("Transformed dvla file for job {} has been removed from s3.".format(job.id))
-
-
 # TODO: remove me, i'm not being run by anything
 @notify_celery.task(name="delete_dvla_response_files")
 @statsd(namespace="tasks")

app/config.py

@@ -263,11 +263,6 @@ class Config(object):
             'schedule': crontab(hour=2, minute=0),
             'options': {'queue': QueueNames.PERIODIC}
         },
-        'remove_transformed_dvla_files': {
-            'task': 'remove_transformed_dvla_files',
-            'schedule': crontab(hour=3, minute=40),
-            'options': {'queue': QueueNames.PERIODIC}
-        },
         'remove_sms_email_jobs': {
             'task': 'remove_sms_email_jobs',
             'schedule': crontab(hour=4, minute=0),
@@ -275,7 +270,7 @@ class Config(object):
         },
         'remove_letter_jobs': {
             'task': 'remove_letter_jobs',
-            'schedule': crontab(hour=4, minute=20),  # this has to run AFTER remove_transformed_dvla_files
+            'schedule': crontab(hour=4, minute=20),
             # since we mark jobs as archived
             'options': {'queue': QueueNames.PERIODIC},
         },
@@ -330,11 +325,6 @@ class Config(object):
 
     SIMULATED_SMS_NUMBERS = ('+447700900000', '+447700900111', '+447700900222')
 
-    DVLA_BUCKETS = {
-        'job': '{}-dvla-file-per-job'.format(os.getenv('NOTIFY_ENVIRONMENT')),
-        'notification': '{}-dvla-letter-api-files'.format(os.getenv('NOTIFY_ENVIRONMENT'))
-    }
-
     FREE_SMS_TIER_FRAGMENT_COUNT = 250000
 
     SMS_INBOUND_WHITELIST = json.loads(os.environ.get('SMS_INBOUND_WHITELIST', '[]'))

tests/app/aws/test_s3.py

@@ -2,7 +2,6 @@ from unittest.mock import call
 from datetime import datetime, timedelta
 import pytest
 import pytz
-from flask import current_app
 
 from freezegun import freeze_time
 
@@ -10,7 +9,6 @@ from app.aws.s3 import (
     get_s3_bucket_objects,
     get_s3_file,
     filter_s3_bucket_objects_within_date_range,
-    remove_transformed_dvla_file,
     get_list_of_files_by_suffix,
 )
 from tests.app.conftest import datetime_in_past
@@ -34,18 +32,6 @@ def test_get_s3_file_makes_correct_call(notify_api, mocker):
     )
 
 
-def test_remove_transformed_dvla_file_makes_correct_call(notify_api, mocker):
-    s3_mock = mocker.patch('app.aws.s3.get_s3_object')
-    fake_uuid = '5fbf9799-6b9b-4dbb-9a4e-74a939f3bb49'
-
-    remove_transformed_dvla_file(fake_uuid)
-
-    s3_mock.assert_has_calls([
-        call(current_app.config['DVLA_BUCKETS']['job'], '{}-dvla-job.text'.format(fake_uuid)),
-        call().delete()
-    ])
-
-
 def test_get_s3_bucket_objects_make_correct_pagination_call(notify_api, mocker):
     paginator_mock = mocker.patch('app.aws.s3.client')
 

tests/app/celery/test_nightly_tasks.py

@@ -1,5 +1,4 @@
 from datetime import datetime, timedelta, date
-from functools import partial
 from unittest.mock import call, patch, PropertyMock
 
 import pytest
@@ -18,7 +17,6 @@ from app.celery.nightly_tasks import (
     raise_alert_if_letter_notifications_still_sending,
     remove_letter_csv_files,
     remove_sms_email_csv_files,
-    remove_transformed_dvla_files,
     s3,
     send_daily_performance_platform_stats,
     send_total_sent_notifications_to_performance_platform,
@@ -291,63 +289,6 @@ def test_should_call_delete_inbound_sms(notify_api, mocker):
     assert nightly_tasks.delete_inbound_sms_older_than_retention.call_count == 1
 
 
-@freeze_time('2017-01-01 10:00:00')
-def test_remove_dvla_transformed_files_removes_expected_files(mocker, sample_service):
-    mocker.patch('app.celery.nightly_tasks.s3.remove_transformed_dvla_file')
-
-    letter_template = create_template(service=sample_service, template_type=LETTER_TYPE)
-
-    job = partial(create_job, template=letter_template)
-
-    seven_days_ago = datetime.utcnow() - timedelta(days=7)
-    just_under_seven_days = seven_days_ago + timedelta(seconds=1)
-    just_over_seven_days = seven_days_ago - timedelta(seconds=1)
-    eight_days_ago = seven_days_ago - timedelta(days=1)
-    nine_days_ago = eight_days_ago - timedelta(days=1)
-    ten_days_ago = nine_days_ago - timedelta(days=1)
-    just_under_nine_days = nine_days_ago + timedelta(seconds=1)
-    just_over_nine_days = nine_days_ago - timedelta(seconds=1)
-    just_over_ten_days = ten_days_ago - timedelta(seconds=1)
-
-    job(created_at=just_under_seven_days)
-    job(created_at=just_over_seven_days)
-    job_to_delete_1 = job(created_at=eight_days_ago)
-    job_to_delete_2 = job(created_at=nine_days_ago)
-    job_to_delete_3 = job(created_at=just_under_nine_days)
-    job_to_delete_4 = job(created_at=just_over_nine_days)
-    job(created_at=just_over_ten_days)
-    remove_transformed_dvla_files()
-
-    s3.remove_transformed_dvla_file.assert_has_calls([
-        call(job_to_delete_1.id),
-        call(job_to_delete_2.id),
-        call(job_to_delete_3.id),
-        call(job_to_delete_4.id),
-    ], any_order=True)
-
-
-def test_remove_dvla_transformed_files_does_not_remove_files(mocker, sample_service):
-    mocker.patch('app.celery.nightly_tasks.s3.remove_transformed_dvla_file')
-
-    letter_template = create_template(service=sample_service, template_type=LETTER_TYPE)
-
-    job = partial(create_job, template=letter_template)
-
-    yesterday = datetime.utcnow() - timedelta(days=1)
-    six_days_ago = datetime.utcnow() - timedelta(days=6)
-    seven_days_ago = six_days_ago - timedelta(days=1)
-    just_over_nine_days = seven_days_ago - timedelta(days=2, seconds=1)
-
-    job(created_at=yesterday)
-    job(created_at=six_days_ago)
-    job(created_at=seven_days_ago)
-    job(created_at=just_over_nine_days)
-
-    remove_transformed_dvla_files()
-
-    s3.remove_transformed_dvla_file.assert_has_calls([])
-
-
 @freeze_time("2016-01-01 11:00:00")
 def test_delete_dvla_response_files_older_than_seven_days_removes_old_files(notify_api, mocker):
     AFTER_SEVEN_DAYS = datetime_in_past(days=8)