Merge pull request #1032 from alphagov/imdad-feat-remove-dvla-response-files

Remove DVLA response files older than 7 days

app/aws/s3.py

@@ -1,6 +1,10 @@
-from boto3 import resource
+from datetime import datetime, timedelta
+
 from flask import current_app
 
+import pytz
+from boto3 import client, resource
+
 FILE_LOCATION_STRUCTURE = 'service-{}-notify/{}.csv'
 
 
@@ -24,7 +28,46 @@ def get_job_from_s3(service_id, job_id):
 def remove_job_from_s3(service_id, job_id):
     bucket_name = current_app.config['CSV_UPLOAD_BUCKET_NAME']
     file_location = FILE_LOCATION_STRUCTURE.format(service_id, job_id)
-    obj = get_s3_object(bucket_name, file_location)
+    return remove_s3_object(bucket_name, file_location)
+
+
+def get_s3_bucket_objects(bucket_name, subfolder='', older_than=7, limit_days=2):
+    boto_client = client('s3', current_app.config['AWS_REGION'])
+    paginator = boto_client.get_paginator('list_objects_v2')
+    page_iterator = paginator.paginate(
+        Bucket=bucket_name,
+        Prefix=subfolder
+    )
+
+    all_objects_in_bucket = []
+    for page in page_iterator:
+        all_objects_in_bucket.extend(page['Contents'])
+
+    return all_objects_in_bucket
+
+
+def filter_s3_bucket_objects_within_date_range(bucket_objects, older_than=7, limit_days=2):
+    """
+    S3 returns the Object['LastModified'] as an 'offset-aware' timestamp so the
+    date range filter must take this into account.
+
+    Additionally an additional Object is returned by S3 corresponding to the
+    container directory. This is redundant and should be removed.
+
+    """
+    end_date = datetime.now(tz=pytz.utc) - timedelta(days=older_than)
+    start_date = end_date - timedelta(days=limit_days)
+    filtered_items = [item for item in bucket_objects if all([
+        not item['Key'].endswith('/'),
+        item['LastModified'] > start_date,
+        item['LastModified'] < end_date
+    ])]
+
+    return filtered_items
+
+
+def remove_s3_object(bucket_name, object_key):
+    obj = get_s3_object(bucket_name, object_key)
     return obj.delete()
 
 

app/celery/scheduled_tasks.py

@@ -255,3 +255,29 @@ def remove_transformed_dvla_files():
     for job in jobs:
         s3.remove_transformed_dvla_file(job.id)
         current_app.logger.info("Transformed dvla file for job {} has been removed from s3.".format(job.id))
+
+
+@notify_celery.task(name="delete_dvla_response_files")
+@statsd(namespace="tasks")
+def delete_dvla_response_files_older_than_seven_days():
+    try:
+        start = datetime.utcnow()
+        bucket_objects = s3.get_s3_bucket_objects(
+            current_app.config['DVLA_RESPONSE_BUCKET_NAME'],
+            'root/dispatch'
+        )
+        older_than_seven_days = s3.filter_s3_bucket_objects_within_date_range(bucket_objects)
+
+        for f in older_than_seven_days:
+            s3.remove_s3_object(current_app.config['DVLA_RESPONSE_BUCKET_NAME'], f['Key'])
+
+        current_app.logger.info(
+            "Delete dvla response files started {} finished {} deleted {} files".format(
+                start,
+                datetime.utcnow(),
+                len(older_than_seven_days)
+            )
+        )
+    except SQLAlchemyError as e:
+        current_app.logger.exception("Failed to delete dvla response files")
+        raise

app/config.py

@@ -1,7 +1,8 @@
 from datetime import timedelta
+import os
+
 from celery.schedules import crontab
 from kombu import Exchange, Queue
-import os
 
 from app.models import (
     EMAIL_TYPE, SMS_TYPE, LETTER_TYPE,
@@ -209,6 +210,11 @@ class Config(object):
             'schedule': crontab(minute=40, hour=4),
             'options': {'queue': QueueNames.PERIODIC}
         },
+        'delete_dvla_response_files': {
+            'task': 'delete_dvla_response_files',
+            'schedule': crontab(minute=10, hour=5),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
         'timeout-job-statistics': {
             'task': 'timeout-job-statistics',
             'schedule': crontab(minute=0, hour=5),
@@ -265,6 +271,7 @@ class Development(Config):
     SQLALCHEMY_ECHO = False
     NOTIFY_EMAIL_DOMAIN = 'notify.tools'
     CSV_UPLOAD_BUCKET_NAME = 'development-notifications-csv-upload'
+    DVLA_RESPONSE_BUCKET_NAME = 'notify.tools-ftp'
     NOTIFY_ENVIRONMENT = 'development'
     NOTIFICATION_QUEUE_PREFIX = 'development'
     DEBUG = True
@@ -284,6 +291,7 @@ class Test(Config):
     NOTIFY_ENVIRONMENT = 'test'
     DEBUG = True
     CSV_UPLOAD_BUCKET_NAME = 'test-notifications-csv-upload'
+    DVLA_RESPONSE_BUCKET_NAME = 'test.notify.com-ftp'
     STATSD_ENABLED = True
     STATSD_HOST = "localhost"
     STATSD_PORT = 1000
@@ -316,6 +324,7 @@ class Preview(Config):
     NOTIFY_EMAIL_DOMAIN = 'notify.works'
     NOTIFY_ENVIRONMENT = 'preview'
     CSV_UPLOAD_BUCKET_NAME = 'preview-notifications-csv-upload'
+    DVLA_RESPONSE_BUCKET_NAME = 'notify.works-ftp'
     FROM_NUMBER = 'preview'
     API_RATE_LIMIT_ENABLED = True
 
@@ -324,6 +333,7 @@ class Staging(Config):
     NOTIFY_EMAIL_DOMAIN = 'staging-notify.works'
     NOTIFY_ENVIRONMENT = 'staging'
     CSV_UPLOAD_BUCKET_NAME = 'staging-notify-csv-upload'
+    DVLA_RESPONSE_BUCKET_NAME = 'staging-notify.works-ftp'
     STATSD_ENABLED = True
     FROM_NUMBER = 'stage'
     API_RATE_LIMIT_ENABLED = True
@@ -333,6 +343,7 @@ class Live(Config):
     NOTIFY_EMAIL_DOMAIN = 'notifications.service.gov.uk'
     NOTIFY_ENVIRONMENT = 'live'
     CSV_UPLOAD_BUCKET_NAME = 'live-notifications-csv-upload'
+    DVLA_RESPONSE_BUCKET_NAME = 'notifications.service.gov.uk-ftp'
     STATSD_ENABLED = True
     FROM_NUMBER = 'GOVUK'
     FUNCTIONAL_TEST_PROVIDER_SERVICE_ID = '6c1d81bb-dae2-4ee9-80b0-89a4aae9f649'
@@ -350,6 +361,7 @@ class Sandbox(CloudFoundryConfig):
     NOTIFY_EMAIL_DOMAIN = 'notify.works'
     NOTIFY_ENVIRONMENT = 'sandbox'
     CSV_UPLOAD_BUCKET_NAME = 'cf-sandbox-notifications-csv-upload'
+    DVLA_RESPONSE_BUCKET_NAME = 'notify.works-ftp'
     FROM_NUMBER = 'sandbox'
     REDIS_ENABLED = False