From 16fd12be6aa1ed9daf63af28fe4d24c3696ed836 Mon Sep 17 00:00:00 2001 From: Rebecca Law Date: Wed, 7 Aug 2019 16:54:04 +0100 Subject: [PATCH] If a job was created 7 days ago but scheduled to send 2 days later, the csv file will be deleted from S3 two days early. For example if a file is uploaded on Aug 1, but scheduled for Aug 3, the csv file will be deleted 2 days before the notifications. This will cause an error for on the jobs pages if the download report link is clicked. --- app/dao/jobs_dao.py | 4 ++-- pytest.ini | 2 +- tests/app/dao/test_jobs_dao.py | 18 ++++++++++++++++++ 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/app/dao/jobs_dao.py b/app/dao/jobs_dao.py index b10174b9f..4427129a4 100644 --- a/app/dao/jobs_dao.py +++ b/app/dao/jobs_dao.py @@ -134,7 +134,7 @@ def dao_get_jobs_older_than_data_retention(notification_types): end_date = today - timedelta(days=f.days_of_retention) jobs.extend(Job.query.join(Template).filter( - Job.created_at < end_date, + func.coalesce(Job.scheduled_for, Job.created_at) < end_date, Job.archived == False, # noqa Template.template_type == f.notification_type, Job.service_id == f.service_id @@ -146,7 +146,7 @@ def dao_get_jobs_older_than_data_retention(notification_types): x.service_id for x in flexible_data_retention if x.notification_type == notification_type ] jobs.extend(Job.query.join(Template).filter( - Job.created_at < end_date, + func.coalesce(Job.scheduled_for, Job.created_at) < end_date, Job.archived == False, # noqa Template.template_type == notification_type, Job.service_id.notin_(services_with_data_retention) diff --git a/pytest.ini b/pytest.ini index a6b0dfd44..fe4669c8b 100644 --- a/pytest.ini +++ b/pytest.ini @@ -6,4 +6,4 @@ env = LOADTESTING_API_KEY=loadtesting FIRETEXT_API_KEY=Firetext NOTIFICATION_QUEUE_PREFIX=testing -addopts = -n 4 -v -p no:warnings +addopts = -v -p no:warnings diff --git a/tests/app/dao/test_jobs_dao.py b/tests/app/dao/test_jobs_dao.py index 5b77664bc..3b533b263 100644 --- a/tests/app/dao/test_jobs_dao.py +++ b/tests/app/dao/test_jobs_dao.py @@ -297,6 +297,24 @@ def test_should_get_jobs_seven_days_old_filters_type(sample_service): assert job_to_remain.id not in [job.id for job in jobs] +@freeze_time('2016-10-31 10:00:00') +def test_should_get_jobs_seven_days_old_by_scheduled_for_date(sample_service): + six_days_ago = datetime.utcnow() - timedelta(days=6) + eight_days_ago = datetime.utcnow() - timedelta(days=8) + letter_template = create_template(sample_service, template_type=LETTER_TYPE) + + create_job(letter_template, created_at=eight_days_ago) + create_job(letter_template, created_at=eight_days_ago, scheduled_for=eight_days_ago) + job_to_remain = create_job(letter_template, created_at=eight_days_ago, scheduled_for=six_days_ago) + + jobs = dao_get_jobs_older_than_data_retention( + notification_types=[LETTER_TYPE] + ) + + assert len(jobs) == 2 + assert job_to_remain.id not in [job.id for job in jobs] + + def assert_job_stat(job, result, sent, delivered, failed): assert result.job_id == job.id assert result.original_file_name == job.original_file_name