remove jobs from letter api and make success/error ftp callback tasks

1. No longer create jobs when creating letters from api 🎉
2. Bulk update notifications based on the notification references after
   we send them to DVLA - either as success or as error

app/celery/tasks.py

@@ -27,7 +27,11 @@ from app.dao.jobs_dao import (
     all_notifications_are_created_for_job,
     dao_get_all_notifications_for_job,
     dao_update_job_status)
-from app.dao.notifications_dao import get_notification_by_id, dao_update_notifications_for_job_to_sent_to_dvla
+from app.dao.notifications_dao import (
+    get_notification_by_id,
+    dao_update_notifications_for_job_to_sent_to_dvla,
+    dao_update_notifications_by_reference
+)
 from app.dao.provider_details_dao import get_current_provider
 from app.dao.service_inbound_api_dao import get_service_inbound_api_for_service
 from app.dao.services_dao import dao_fetch_service_by_id, fetch_todays_total_message_count
@@ -309,17 +313,6 @@ def update_job_to_sent_to_dvla(self, job_id):
                             "Updated {} job to {}".format(updated_count, job_id, JOB_STATUS_SENT_TO_DVLA))
 
 
-@notify_celery.task(bind=True, name='update-notifications-to-sent')
-@statsd(namespace="tasks")
-def update_notifications_to_sent_to_dvla(self, notification_references):
-    # This task will be called by the FTP app to update notifications as sent to DVLA
-    provider = get_current_provider(LETTER_TYPE)
-
-    updated_count = dao_update_notifications_for_job_to_sent_to_dvla(references, provider.identifier)
-
-    current_app.logger.info("Updated {} letter notifications to sending. ".format(updated_count))
-
-
 @notify_celery.task(bind=True, name='update-letter-job-to-error')
 @statsd(namespace="tasks")
 def update_dvla_job_to_error(self, job_id):
@@ -327,6 +320,42 @@ def update_dvla_job_to_error(self, job_id):
     current_app.logger.info("Updated {} job to {}".format(job_id, JOB_STATUS_ERROR))
 
 
+@notify_celery.task(bind=True, name='update-letter-notifications-to-sent')
+@statsd(namespace="tasks")
+def update_letter_notifications_to_sent_to_dvla(self, notification_references):
+    # This task will be called by the FTP app to update notifications as sent to DVLA
+    provider = get_current_provider(LETTER_TYPE)
+
+    updated_count = dao_update_notifications_by_reference(
+        notification_references,
+        {
+            'status': NOTIFICATION_SENDING,
+            'sent_by': provider.identifier,
+            'sent_at': datetime.utcnow(),
+            'updated_at': datetime.utcnow()
+        }
+    )
+
+    current_app.logger.info("Updated {} letter notifications to sending".format(updated_count))
+
+
+@notify_celery.task(bind=True, name='update-letter-notifications-to-error')
+@statsd(namespace="tasks")
+def update_letter_notifications_to_error(self, notification_references):
+    # This task will be called by the FTP app to update notifications as sent to DVLA
+    provider = get_current_provider(LETTER_TYPE)
+
+    updated_count = dao_update_notifications_by_reference(
+        notification_references,
+        {
+            'status': NOTIFICATION_TECHNICAL_FAILURE,
+            'updated_at': datetime.utcnow()
+        }
+    )
+
+    current_app.logger.info("Updated {} letter notifications to technical-failure".format(updated_count))
+
+
 def create_dvla_file_contents_for_job(job_id):
     notifications = dao_get_all_notifications_for_job(job_id)
 

app/config.py

@@ -260,7 +260,7 @@ class Config(object):
 
     DVLA_BUCKETS = {
         'job': '{}-dvla-file-per-job'.format(os.getenv('NOTIFY_ENVIRONMENT')),
-        'notification': '{}-dvla-file-per-job'.format(os.getenv('NOTIFY_ENVIRONMENT'))
+        'notification': '{}-dvla-file-letter-api'.format(os.getenv('NOTIFY_ENVIRONMENT'))
     }
 
     API_KEY_LIMITS = {

app/dao/notifications_dao.py

@@ -475,15 +475,19 @@ def dao_update_notifications_for_job_to_sent_to_dvla(job_id, provider):
 
 @statsd(namespace="dao")
 @transactional
-def dao_update_notifications_by_reference_sent_to_dvla(notification_references, provider):
+def dao_update_notifications_by_reference(references, update_dict):
     now = datetime.utcnow()
-    updated_count = db.session.query(
+    updated_count = Notification.query().filter(
-        Notification).filter(Notification.job_id == job_id).update(
+        Notification.reference.in_(references)
-        {'status': NOTIFICATION_SENDING, "sent_by": provider, "sent_at": now})
+    ).update(
+        update_dict
+    )
 
-    db.session.query(
+    NotificationHistory.query().filter(
-        NotificationHistory).filter(NotificationHistory.job_id == job_id).update(
+        NotificationHistory.reference.in_(references)
-        {'status': NOTIFICATION_SENDING, "sent_by": provider, "sent_at": now, "updated_at": now})
+    ).update(
+        update_dict
+    )
 
     return updated_count
 

app/notifications/process_letter_notifications.py

@@ -26,10 +26,10 @@ def create_letter_api_job(template):
     return job
 
 
-def create_letter_notification(letter_data, job, api_key):
+def create_letter_notification(letter_data, template, api_key):
     notification = persist_notification(
-        template_id=job.template.id,
+        template_id=template.id,
-        template_version=job.template.version,
+        template_version=template.version,
         # we only accept addresses_with_underscores from the API (from CSV we also accept dashes, spaces etc)
         recipient=letter_data['personalisation']['address_line_1'],
         service=job.service,
@@ -37,8 +37,8 @@ def create_letter_notification(letter_data, job, api_key):
         notification_type=LETTER_TYPE,
         api_key_id=api_key.id,
         key_type=api_key.key_type,
-        job_id=job.id,
+        job_id=None,
-        job_row_number=0,
+        job_row_number=None,
         reference=create_random_identifier(),
         client_reference=letter_data.get('reference')
     )

app/v2/notifications/post_notifications.py

@@ -153,8 +153,7 @@ def process_letter_notification(*, letter_data, api_key, template):
     if api_key.service.restricted and api_key.key_type != KEY_TYPE_TEST:
         raise BadRequestError(message='Cannot send letters when service is in trial mode', status_code=403)
 
-    job = create_letter_api_job(template)
+    notification = create_letter_notification(letter_data, template, api_key)
-    notification = create_letter_notification(letter_data, job, api_key)
 
     if api_key.service.research_mode or api_key.key_type == KEY_TYPE_TEST:
         # distinguish real API jobs from test jobs by giving the test jobs a different filename