remove jobs from letter api calls

we now no longer create a job. At the end of the post there is no
action, as we don't have any tasks to queue immediately - if it's a
real notification it'll get picked up in the evening scheduled task.

If it's a test notification, we create it with an initial status of
sending so that we can be sure it'll never get picked up - and then we
trigger the update-letter-notifications-to-sent-to-dvla task to sent
the sent-at/by.

app/celery/scheduled_tasks.py

@@ -335,17 +335,17 @@ def run_letter_notifications():
     if notifications:
         file_contents = create_dvla_file_contents_for_notifications(notifications)
 
-        file_location = '{}-dvla-notifications.txt'.format(current_time)
+        filename = '{}-dvla-notifications.txt'.format(current_time)
         s3upload(
             filedata=file_contents + '\n',
             region=current_app.config['AWS_REGION'],
             bucket_name=current_app.config['DVLA_BUCKETS']['notification'],
-            file_location=file_location
+            file_location=filename
         )
 
         notify_celery.send_task(
             name=TaskNames.DVLA_NOTIFICATIONS,
-            kwargs={'filename': file_location},
+            kwargs={'filename': filename},
             queue=QueueNames.PROCESS_FTP
         )
         current_app.logger.info(

app/celery/tasks.py

@@ -46,7 +46,10 @@ from app.models import (
     JOB_STATUS_IN_PROGRESS,
     JOB_STATUS_FINISHED,
     JOB_STATUS_READY_TO_SEND,
-    JOB_STATUS_SENT_TO_DVLA, JOB_STATUS_ERROR)
+    JOB_STATUS_SENT_TO_DVLA, JOB_STATUS_ERROR,
+    NOTIFICATION_SENDING,
+    NOTIFICATION_TECHNICAL_FAILURE
+)
 from app.notifications.process_notifications import persist_notification
 from app.service.utils import service_allowed_to_send_to
 from app.statsd_decorators import statsd
@@ -343,7 +346,6 @@ def update_letter_notifications_to_sent_to_dvla(self, notification_references):
 @statsd(namespace="tasks")
 def update_letter_notifications_to_error(self, notification_references):
     # This task will be called by the FTP app to update notifications as sent to DVLA
-    provider = get_current_provider(LETTER_TYPE)
 
     updated_count = dao_update_notifications_by_reference(
         notification_references,

app/dao/notifications_dao.py

@@ -477,16 +477,18 @@ def dao_update_notifications_for_job_to_sent_to_dvla(job_id, provider):
 @transactional
 def dao_update_notifications_by_reference(references, update_dict):
     now = datetime.utcnow()
-    updated_count = Notification.query().filter(
+    updated_count = Notification.query.filter(
         Notification.reference.in_(references)
     ).update(
-        update_dict
+        update_dict,
+        synchronize_session=False
     )
 
-    NotificationHistory.query().filter(
+    NotificationHistory.query.filter(
         NotificationHistory.reference.in_(references)
     ).update(
-        update_dict
+        update_dict,
+        synchronize_session=False
     )
 
     return updated_count
@@ -585,11 +587,12 @@ def dao_set_created_live_letter_api_notifications_to_pending():
     from completing until it commits.
     """
     return db.session.query(
-        Notification.id
+        Notification
     ).filter(
         Notification.notification_type == LETTER_TYPE,
         Notification.status == NOTIFICATION_CREATED,
         Notification.key_type == KEY_TYPE_NORMAL,
+        Notification.api_key != None
     ).with_for_update(
     ).all()
 

app/notifications/process_letter_notifications.py

@@ -1,38 +1,15 @@
 from app import create_random_identifier
-from app.models import LETTER_TYPE, JOB_STATUS_READY_TO_SEND, Job
+from app.models import LETTER_TYPE
-from app.dao.jobs_dao import dao_create_job
 from app.notifications.process_notifications import persist_notification
-from app.v2.errors import InvalidRequest
-from app.variables import LETTER_API_FILENAME
 
 
-def create_letter_api_job(template):
+def create_letter_notification(letter_data, template, api_key, status):
-    service = template.service
-    if not service.active:
-        raise InvalidRequest('Service {} is inactive'.format(service.id), 403)
-    if template.archived:
-        raise InvalidRequest('Template {} is deleted'.format(template.id), 400)
-
-    job = Job(
-        original_file_name=LETTER_API_FILENAME,
-        service=service,
-        template=template,
-        template_version=template.version,
-        notification_count=1,
-        job_status=JOB_STATUS_READY_TO_SEND,
-        created_by=None
-    )
-    dao_create_job(job)
-    return job
-
-
-def create_letter_notification(letter_data, template, api_key):
     notification = persist_notification(
         template_id=template.id,
         template_version=template.version,
         # we only accept addresses_with_underscores from the API (from CSV we also accept dashes, spaces etc)
         recipient=letter_data['personalisation']['address_line_1'],
-        service=job.service,
+        service=template.service,
         personalisation=letter_data['personalisation'],
         notification_type=LETTER_TYPE,
         api_key_id=api_key.id,
@@ -40,6 +17,7 @@ def create_letter_notification(letter_data, template, api_key):
         job_id=None,
         job_row_number=None,
         reference=create_random_identifier(),
-        client_reference=letter_data.get('reference')
+        client_reference=letter_data.get('reference'),
+        status=status
     )
     return notification

app/notifications/process_notifications.py

@@ -3,6 +3,7 @@ from datetime import datetime
 
 from flask import current_app
 
+from notifications_utils.clients import redis
 from notifications_utils.recipients import (
     get_international_phone_info,
     validate_and_format_phone_number,
@@ -11,10 +12,8 @@ from notifications_utils.recipients import (
 
 from app import redis_store
 from app.celery import provider_tasks
-from notifications_utils.clients import redis
-
 from app.config import QueueNames
-from app.models import SMS_TYPE, Notification, KEY_TYPE_TEST, EMAIL_TYPE, ScheduledNotification
+from app.models import SMS_TYPE, Notification, KEY_TYPE_TEST, EMAIL_TYPE, NOTIFICATION_CREATED, ScheduledNotification
 from app.dao.notifications_dao import (dao_create_notification,
                                        dao_delete_notifications_and_history_by_id,
                                        dao_created_scheduled_notification)
@@ -52,7 +51,8 @@ def persist_notification(
     client_reference=None,
     notification_id=None,
     simulated=False,
-    created_by_id=None
+    created_by_id=None,
+    status=NOTIFICATION_CREATED
 ):
     notification_created_at = created_at or datetime.utcnow()
     if not notification_id:
@@ -73,7 +73,8 @@ def persist_notification(
         job_row_number=job_row_number,
         client_reference=client_reference,
         reference=reference,
-        created_by_id=created_by_id
+        created_by_id=created_by_id,
+        status=status
     )
 
     if notification_type == SMS_TYPE:

app/v2/notifications/post_notifications.py

@@ -4,16 +4,23 @@ from flask import request, jsonify, current_app, abort
 
 from app import api_user, authenticated_service
 from app.config import QueueNames
-from app.dao.jobs_dao import dao_update_job
+from app.models import (
-from app.models import SMS_TYPE, EMAIL_TYPE, LETTER_TYPE, PRIORITY, KEY_TYPE_TEST, KEY_TYPE_TEAM
+    SMS_TYPE,
-from app.celery.tasks import build_dvla_file, update_job_to_sent_to_dvla
+    EMAIL_TYPE,
+    LETTER_TYPE,
+    PRIORITY,
+    KEY_TYPE_TEST,
+    KEY_TYPE_TEAM,
+    NOTIFICATION_CREATED,
+    NOTIFICATION_SENDING
+)
+from app.celery.tasks import update_letter_notifications_to_sent_to_dvla
 from app.notifications.process_notifications import (
     persist_notification,
     send_notification_to_queue,
     simulated_recipient,
     persist_scheduled_notification)
 from app.notifications.process_letter_notifications import (
-    create_letter_api_job,
     create_letter_notification
 )
 from app.notifications.validators import (
@@ -153,18 +160,17 @@ def process_letter_notification(*, letter_data, api_key, template):
     if api_key.service.restricted and api_key.key_type != KEY_TYPE_TEST:
         raise BadRequestError(message='Cannot send letters when service is in trial mode', status_code=403)
 
-    notification = create_letter_notification(letter_data, template, api_key)
+    should_send = not (api_key.service.research_mode or api_key.key_type == KEY_TYPE_TEST)
 
-    if api_key.service.research_mode or api_key.key_type == KEY_TYPE_TEST:
+    # if we don't want to actually send the letter, then start it off in SENDING so we don't pick it up
-        # distinguish real API jobs from test jobs by giving the test jobs a different filename
+    status = NOTIFICATION_CREATED if should_send else NOTIFICATION_SENDING
-        job.original_file_name = LETTER_TEST_API_FILENAME
+
-        dao_update_job(job)
+    notification = create_letter_notification(letter_data, template, api_key, status=status)
-        update_job_to_sent_to_dvla.apply_async([str(job.id)], queue=QueueNames.RESEARCH_MODE)
+
-    else:
+    if not should_send:
-        build_dvla_file.apply_async([str(job.id)], queue=QueueNames.JOBS)
+        update_letter_notifications_to_sent_to_dvla.apply_async(
+            kwargs={'notification_references': [notification.reference]},
+            queue=QueueNames.RESEARCH_MODE
+        )
 
-    current_app.logger.info("send job {} for api notification {} to build-dvla-file in the process-job queue".format(
-        job.id,
-        notification.id
-    ))
     return notification

tests/app/notifications/test_process_letter_notifications.py

@@ -5,7 +5,6 @@ from app.models import Job
 from app.models import JOB_STATUS_READY_TO_SEND
 from app.models import LETTER_TYPE
 from app.models import Notification
-from app.notifications.process_letter_notifications import create_letter_api_job
 from app.notifications.process_letter_notifications import create_letter_notification
 from app.v2.errors import InvalidRequest
 from app.variables import LETTER_API_FILENAME