add run_letter_notifications scheduled task

this task grabs all notifications that are sent via the API, and
are still in created - and sends them off to DVLA.

app/celery/scheduled_tasks.py

@@ -5,6 +5,7 @@ from datetime import (
 
 from flask import current_app
 from sqlalchemy.exc import SQLAlchemyError
+from notifications_utils.s3 import s3upload
 
 from app.aws import s3
 from app import notify_celery
@@ -37,7 +38,7 @@ from app.dao.users_dao import delete_codes_older_created_more_than_a_day_ago
 from app.models import LETTER_TYPE, JOB_STATUS_READY_TO_SEND
 from app.notifications.process_notifications import send_notification_to_queue
 from app.statsd_decorators import statsd
-from app.celery.tasks import process_job
+from app.celery.tasks import process_job, create_dvla_file_contents_for_notifications
 from app.config import QueueNames, TaskNames
 from app.utils import convert_utc_to_bst
 
@@ -319,3 +320,26 @@ def run_letter_jobs():
         queue=QueueNames.PROCESS_FTP
     )
     current_app.logger.info("Queued {} ready letter job ids onto {}".format(len(job_ids), QueueNames.PROCESS_FTP))
+
+
+@notify_celery.task(name="run-letter-notifications")
+@statsd(namespace="tasks")
+def run_letter_notifications():
+    notifications = dao_get_created_letter_api_notifications_that_dont_belong_to_jobs()
+
+    file_contents = create_dvla_file_contents_for_notifications(notifications)
+    s3upload(
+        filedata=file_contents + '\n',
+        region=current_app.config['AWS_REGION'],
+        bucket_name=current_app.config['DVLA_UPLOAD_BUCKET_NAME'],
+        file_location='2017-09-12-dvla-notifications.txt'
+    )
+
+    # set noti statuses to pending or something
+
+    notify_celery.send_task(
+        name=TaskNames.DVLA_NOTIFICATIONS,
+        kwargs={'date': '2017-09-12'},
+        queue=QueueNames.PROCESS_FTP
+    )
+    current_app.logger.info("Queued {} ready letter job ids onto {}".format(len(notifications), QueueNames.PROCESS_FTP))

app/celery/tasks.py

@@ -309,6 +309,10 @@ def update_job_to_sent_to_dvla(self, job_id):
                             "Updated {} job to {}".format(updated_count, job_id, JOB_STATUS_SENT_TO_DVLA))
 
 
+def update_notifications_to_sent_to_dvla(self, notification_references):
+
+
+
 @notify_celery.task(bind=True, name='update-letter-job-to-error')
 @statsd(namespace="tasks")
 def update_dvla_job_to_error(self, job_id):
@@ -316,7 +320,13 @@ def update_dvla_job_to_error(self, job_id):
     current_app.logger.info("Updated {} job to {}".format(job_id, JOB_STATUS_ERROR))
 
 
-def create_dvla_file_contents(job_id):
+def create_dvla_file_contents_for_job(job_id):
+    notifications = dao_get_all_notifications_for_job(job_id)
+
+    return create_dvla_file_contents_for_notifications(notifications)
+
+
+def create_dvla_file_contents_for_notifications(notifications):
     file_contents = '\n'.join(
         str(LetterDVLATemplate(
             notification.template.__dict__,
@@ -325,7 +335,7 @@ def create_dvla_file_contents(job_id):
             contact_block=notification.service.letter_contact_block,
             org_id=notification.service.dvla_organisation.id,
         ))
-        for notification in dao_get_all_notifications_for_job(job_id)
+        for notification in notifications
     )
     return file_contents
 

app/config.py

@@ -229,6 +229,11 @@ class Config(object):
             'task': 'run-letter-jobs',
             'schedule': crontab(hour=17, minute=30),
             'options': {'queue': QueueNames.PERIODIC}
+        },
+        'run-letter-notifications': {
+            'task': 'run-letter-notifications',
+            'schedule': crontab(hour=17, minute=35),
+            'options': {'queue': QueueNames.PERIODIC}
         }
     }
     CELERY_QUEUES = []
@@ -253,7 +258,10 @@ class Config(object):
     FUNCTIONAL_TEST_PROVIDER_SERVICE_ID = None
     FUNCTIONAL_TEST_PROVIDER_SMS_TEMPLATE_ID = None
 
-    DVLA_UPLOAD_BUCKET_NAME = "{}-dvla-file-per-job".format(os.getenv('NOTIFY_ENVIRONMENT'))
+    DVLA_BUCKETS = {
+        'job': '{}-dvla-file-per-job'.format(os.getenv('NOTIFY_ENVIRONMENT')),
+        'notification': '{}-dvla-file-per-job'.format(os.getenv('NOTIFY_ENVIRONMENT'))
+    }
 
     API_KEY_LIMITS = {
         KEY_TYPE_TEAM: {

tests/app/celery/test_tasks.py

@@ -17,7 +17,7 @@ from app.celery import tasks
 from app.celery.tasks import (
     s3,
     build_dvla_file,
-    create_dvla_file_contents,
+    create_dvla_file_contents_for_job,
     update_dvla_job_to_error,
     process_job,
     process_row,
@@ -1081,7 +1081,7 @@ def test_build_dvla_file_retries_if_all_notifications_are_not_created(sample_let
     mocked_send_task.assert_not_called()
 
 
-def test_create_dvla_file_contents(sample_letter_template, mocker):
+def test_create_dvla_file_contents_for_job(sample_letter_template, mocker):
     job = create_job(template=sample_letter_template, notification_count=2)
     create_notification(template=job.template, job=job, reference=1)
     create_notification(template=job.template, job=job, reference=2)
@@ -1089,7 +1089,7 @@ def test_create_dvla_file_contents(sample_letter_template, mocker):
     mocked_letter_template_instance = mocked_letter_template.return_value
     mocked_letter_template_instance.__str__.return_value = "dvla|string"
 
-    create_dvla_file_contents(job.id)
+    create_dvla_file_contents_for_job(job.id)
     calls = mocked_letter_template.call_args_list
     # Template
     assert calls[0][0][0]['subject'] == 'Template subject'