From c4a89b77e38b79e345601187e644a3ad3a83623c Mon Sep 17 00:00:00 2001 From: Imdad Ahad Date: Wed, 7 Jun 2017 16:31:51 +0100 Subject: [PATCH] Add task to remove transformed dvla files: * Runs at 4:40am * Remove files older than 7 days and only two day's worth --- app/celery/scheduled_tasks.py | 10 ++++++ app/config.py | 5 +++ tests/app/celery/test_scheduled_tasks.py | 44 ++++++++++++++++++++++-- 3 files changed, 56 insertions(+), 3 deletions(-) diff --git a/app/celery/scheduled_tasks.py b/app/celery/scheduled_tasks.py index 636522f01..11e4de49d 100644 --- a/app/celery/scheduled_tasks.py +++ b/app/celery/scheduled_tasks.py @@ -24,6 +24,7 @@ from app.dao.provider_details_dao import ( dao_toggle_sms_provider ) from app.dao.users_dao import delete_codes_older_created_more_than_a_day_ago +from app.models import LETTER_TYPE from app.notifications.process_notifications import send_notification_to_queue from app.statsd_decorators import statsd from app.celery.tasks import process_job @@ -245,3 +246,12 @@ def delete_inbound_sms_older_than_seven_days(): except SQLAlchemyError as e: current_app.logger.exception("Failed to delete inbound sms notifications") raise + + +@notify_celery.task(name="remove_transformed_dvla_files") +@statsd(namespace="tasks") +def remove_transformed_dvla_files(): + jobs = dao_get_jobs_older_than_limited_by(job_types=[LETTER_TYPE]) + for job in jobs: + s3.remove_transformed_dvla_file(job.id) + current_app.logger.info("Transformed dvla file for job {} has been removed from s3.".format(job.id)) diff --git a/app/config.py b/app/config.py index cbaa15d67..08f39e656 100644 --- a/app/config.py +++ b/app/config.py @@ -204,6 +204,11 @@ class Config(object): 'options': {'queue': QueueNames.PERIODIC}, 'kwargs': {'job_types': [LETTER_TYPE]} }, + 'remove_transformed_dvla_files': { + 'task': 'remove_transformed_dvla_files', + 'schedule': crontab(minute=40, hour=4), + 'options': {'queue': QueueNames.PERIODIC} + }, 'timeout-job-statistics': { 'task': 'timeout-job-statistics', 'schedule': crontab(minute=0, hour=5), diff --git a/tests/app/celery/test_scheduled_tasks.py b/tests/app/celery/test_scheduled_tasks.py index b3ff5d478..36b9ad668 100644 --- a/tests/app/celery/test_scheduled_tasks.py +++ b/tests/app/celery/test_scheduled_tasks.py @@ -1,10 +1,12 @@ -import pytest - from datetime import datetime, timedelta from functools import partial +from unittest.mock import call, patch, PropertyMock from flask import current_app + +import pytest from freezegun import freeze_time + from app.celery import scheduled_tasks from app.celery.scheduled_tasks import ( delete_email_notifications_older_than_seven_days, @@ -15,6 +17,7 @@ from app.celery.scheduled_tasks import ( delete_sms_notifications_older_than_seven_days, delete_verify_codes, remove_csv_files, + remove_transformed_dvla_files, run_scheduled_jobs, s3, send_daily_performance_platform_stats, @@ -41,7 +44,6 @@ from tests.app.conftest import ( sample_notification_history as create_notification_history, create_custom_template) from tests.conftest import set_config_values -from unittest.mock import call, patch, PropertyMock def _create_slow_delivery_notification(provider='mmg'): @@ -89,6 +91,8 @@ def test_should_have_decorated_tasks_functions(): 'switch_current_sms_provider_on_slow_delivery' assert delete_inbound_sms_older_than_seven_days.__wrapped__.__name__ == \ 'delete_inbound_sms_older_than_seven_days' + assert remove_transformed_dvla_files.__wrapped__.__name__ == \ + 'remove_transformed_dvla_files' def test_should_call_delete_sms_notifications_more_than_week_in_task(notify_api, mocker): @@ -488,3 +492,37 @@ def test_remove_csv_files_filters_by_type(mocker, sample_service): assert s3.remove_job_from_s3.call_args_list == [ call(job_to_delete.service_id, job_to_delete.id), ] + + +@freeze_time('2017-01-01 10:00:00') +def test_remove_dvla_transformed_files_removes_expected_files(mocker, sample_service): + mocker.patch('app.celery.scheduled_tasks.s3.remove_transformed_dvla_file') + + letter_template = create_template(service=sample_service, template_type=LETTER_TYPE) + + job = partial(create_job, template=letter_template) + + seven_days_ago = datetime.utcnow() - timedelta(days=7) + just_under_seven_days = seven_days_ago + timedelta(seconds=1) + just_over_seven_days = seven_days_ago - timedelta(seconds=1) + eight_days_ago = seven_days_ago - timedelta(days=1) + nine_days_ago = eight_days_ago - timedelta(days=1) + just_under_nine_days = nine_days_ago + timedelta(seconds=1) + just_over_nine_days = nine_days_ago - timedelta(seconds=1) + + job(created_at=seven_days_ago) + job(created_at=just_under_seven_days) + job_to_delete_1 = job(created_at=just_over_seven_days) + job_to_delete_2 = job(created_at=eight_days_ago) + job_to_delete_3 = job(created_at=nine_days_ago) + job_to_delete_4 = job(created_at=just_under_nine_days) + job(created_at=just_over_nine_days) + + remove_transformed_dvla_files() + + s3.remove_transformed_dvla_file.assert_has_calls([ + call(job_to_delete_1.id), + call(job_to_delete_2.id), + call(job_to_delete_3.id), + call(job_to_delete_4.id), + ], any_order=True)