Merge pull request #1136 from alphagov/letter-api

Letter api

app/dao/jobs_dao.py

@@ -1,3 +1,4 @@
+import uuid
 from datetime import datetime, timedelta
 
 from flask import current_app
@@ -10,6 +11,7 @@ from app.models import (
     JOB_STATUS_SCHEDULED, JOB_STATUS_PENDING,
     LETTER_TYPE
 )
+from app.variables import LETTER_TEST_API_FILENAME
 from app.statsd_decorators import statsd
 
 
@@ -108,6 +110,8 @@ def dao_get_future_scheduled_job_by_id_and_service_id(job_id, service_id):
 
 
 def dao_create_job(job):
+    if not job.id:
+        job.id = uuid.uuid4()
     job_stats = JobStatistics(
         job_id=job.id,
         updated_at=datetime.utcnow()
@@ -139,9 +143,18 @@ def dao_get_jobs_older_than_limited_by(job_types, older_than=7, limit_days=2):
 
 
 def dao_get_all_letter_jobs():
-    return db.session.query(Job).join(Job.template).filter(
-        Template.template_type == LETTER_TYPE
-    ).order_by(desc(Job.created_at)).all()
+    return db.session.query(
+        Job
+    ).join(
+        Job.template
+    ).filter(
+        Template.template_type == LETTER_TYPE,
+        # test letter jobs (or from research mode services) are created with a different filename,
+        # exclude them so we don't see them on the send to CSV
+        Job.original_file_name != LETTER_TEST_API_FILENAME
+    ).order_by(
+        desc(Job.created_at)
+    ).all()
 
 
 @statsd(namespace="dao")

app/dao/notifications_dao.py

@@ -338,7 +338,8 @@ def get_notifications_for_service(
         filters.append(Notification.created_at < older_than_created_at)
 
     if not include_jobs or (key_type and key_type != KEY_TYPE_NORMAL):
-        filters.append(Notification.job_id.is_(None))
+        # we can't say "job_id == None" here, because letters sent via the API still have a job_id :(
+        filters.append(Notification.api_key_id != None)  # noqa
 
     if key_type is not None:
         filters.append(Notification.key_type == key_type)

app/dao/services_dao.py

@@ -207,14 +207,14 @@ def delete_service_and_all_associated_db_objects(service):
     _delete_commit(ProviderStatistics.query.filter_by(service=service))
     _delete_commit(InvitedUser.query.filter_by(service=service))
     _delete_commit(Permission.query.filter_by(service=service))
-    _delete_commit(ApiKey.query.filter_by(service=service))
-    _delete_commit(ApiKey.get_history_model().query.filter_by(service_id=service.id))
     _delete_commit(NotificationHistory.query.filter_by(service=service))
     _delete_commit(Notification.query.filter_by(service=service))
     _delete_commit(Job.query.filter_by(service=service))
     _delete_commit(Template.query.filter_by(service=service))
     _delete_commit(TemplateHistory.query.filter_by(service_id=service.id))
     _delete_commit(ServicePermission.query.filter_by(service_id=service.id))
+    _delete_commit(ApiKey.query.filter_by(service=service))
+    _delete_commit(ApiKey.get_history_model().query.filter_by(service_id=service.id))
 
     verify_codes = VerifyCode.query.join(User).filter(User.id.in_([x.id for x in service.users]))
     list(map(db.session.delete, verify_codes))

app/models.py

@@ -619,7 +619,7 @@ class JobStatus(db.Model):
 class Job(db.Model):
     __tablename__ = 'jobs'
 
-    id = db.Column(UUID(as_uuid=True), primary_key=True)
+    id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
     original_file_name = db.Column(db.String, nullable=False)
     service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'), index=True, unique=False, nullable=False)
     service = db.relationship('Service', backref=db.backref('jobs', lazy='dynamic'))
@@ -654,7 +654,7 @@ class Job(db.Model):
         unique=False,
         nullable=True)
     created_by = db.relationship('User')
-    created_by_id = db.Column(UUID(as_uuid=True), db.ForeignKey('users.id'), index=True, nullable=False)
+    created_by_id = db.Column(UUID(as_uuid=True), db.ForeignKey('users.id'), index=True, nullable=True)
     scheduled_for = db.Column(
         db.DateTime,
         index=True,
@@ -891,7 +891,7 @@ class Notification(db.Model):
     @property
     def subject(self):
         from app.utils import get_template_instance
-        if self.notification_type == EMAIL_TYPE:
+        if self.notification_type != SMS_TYPE:
             template_object = get_template_instance(self.template.__dict__, self.personalisation)
             return template_object.subject
 
@@ -971,10 +971,24 @@ class Notification(db.Model):
             "created_at": self.created_at.strftime(DATETIME_FORMAT),
             "sent_at": self.sent_at.strftime(DATETIME_FORMAT) if self.sent_at else None,
             "completed_at": self.completed_at(),
-            "scheduled_for": convert_bst_to_utc(self.scheduled_notification.scheduled_for
-                                                ).strftime(DATETIME_FORMAT) if self.scheduled_notification else None
+            "scheduled_for": (
+                convert_bst_to_utc(
+                    self.scheduled_notification.scheduled_for
+                ).strftime(DATETIME_FORMAT)
+                if self.scheduled_notification
+                else None
+            )
         }
 
+        if self.notification_type == LETTER_TYPE:
+            serialized['line_1'] = self.personalisation['address_line_1']
+            serialized['line_2'] = self.personalisation.get('address_line_2')
+            serialized['line_3'] = self.personalisation.get('address_line_3')
+            serialized['line_4'] = self.personalisation.get('address_line_4')
+            serialized['line_5'] = self.personalisation.get('address_line_5')
+            serialized['line_6'] = self.personalisation.get('address_line_6')
+            serialized['postcode'] = self.personalisation['postcode']
+
         return serialized
 
 

app/notifications/process_letter_notifications.py

@@ -0,0 +1,45 @@
+from app import create_random_identifier
+from app.models import LETTER_TYPE, JOB_STATUS_READY_TO_SEND, Job
+from app.dao.jobs_dao import dao_create_job
+from app.notifications.process_notifications import persist_notification
+from app.v2.errors import InvalidRequest
+from app.variables import LETTER_API_FILENAME
+
+
+def create_letter_api_job(template):
+    service = template.service
+    if not service.active:
+        raise InvalidRequest('Service {} is inactive'.format(service.id), 403)
+    if template.archived:
+        raise InvalidRequest('Template {} is deleted'.format(template.id), 400)
+
+    job = Job(
+        original_file_name=LETTER_API_FILENAME,
+        service=service,
+        template=template,
+        template_version=template.version,
+        notification_count=1,
+        job_status=JOB_STATUS_READY_TO_SEND,
+        created_by=None
+    )
+    dao_create_job(job)
+    return job
+
+
+def create_letter_notification(letter_data, job, api_key):
+    notification = persist_notification(
+        template_id=job.template.id,
+        template_version=job.template.version,
+        # we only accept addresses_with_underscores from the API (from CSV we also accept dashes, spaces etc)
+        recipient=letter_data['personalisation']['address_line_1'],
+        service=job.service,
+        personalisation=letter_data['personalisation'],
+        notification_type=LETTER_TYPE,
+        api_key_id=api_key.id,
+        key_type=api_key.key_type,
+        job_id=job.id,
+        job_row_number=0,
+        reference=create_random_identifier(),
+        client_reference=letter_data.get('reference')
+    )
+    return notification

app/notifications/process_notifications.py

@@ -55,7 +55,7 @@ def persist_notification(
     created_by_id=None
 ):
     notification_created_at = created_at or datetime.utcnow()
-    if not notification_id and simulated:
+    if not notification_id:
         notification_id = uuid.uuid4()
     notification = Notification(
         id=notification_id,

app/schema_validation/definitions.py

@@ -14,7 +14,6 @@ uuid = {
 
 personalisation = {
     "type": "object",
-    "validationMessage": "should contain key value pairs",
     "code": "1001",  # yet to be implemented
     "link": "link to our error documentation not yet implemented"
 }

app/utils.py

@@ -30,7 +30,7 @@ def url_with_token(data, url, config):
 def get_template_instance(template, values):
     from app.models import SMS_TYPE, EMAIL_TYPE, LETTER_TYPE
     return {
-        SMS_TYPE: SMSMessageTemplate, EMAIL_TYPE: PlainTextEmailTemplate, LETTER_TYPE: LetterPreviewTemplate
+        SMS_TYPE: SMSMessageTemplate, EMAIL_TYPE: PlainTextEmailTemplate, LETTER_TYPE: PlainTextEmailTemplate
     }[template['template_type']](template, values)
 
 

app/v2/errors.py

@@ -29,10 +29,10 @@ class RateLimitError(InvalidRequest):
 
 
 class BadRequestError(InvalidRequest):
-    status_code = 400
     message = "An error occurred"
 
-    def __init__(self, fields=[], message=None):
+    def __init__(self, fields=[], message=None, status_code=400):
+        self.status_code = status_code
         self.fields = fields
         self.message = message if message else self.message
 

app/v2/notifications/notification_schemas.py

@@ -228,7 +228,8 @@ post_letter_response = {
         "content": letter_content,
         "uri": {"type": "string", "format": "uri"},
         "template": template,
-        "scheduled_for": {"type": ["string", "null"]}
+        # letters cannot be scheduled
+        "scheduled_for": {"type": "null"}
     },
     "required": ["id", "content", "uri", "template"]
 }

app/v2/notifications/post_notifications.py

@@ -4,12 +4,18 @@ from flask import request, jsonify, current_app, abort
 
 from app import api_user, authenticated_service
 from app.config import QueueNames
-from app.models import SMS_TYPE, EMAIL_TYPE, LETTER_TYPE, PRIORITY
+from app.dao.jobs_dao import dao_update_job
+from app.models import SMS_TYPE, EMAIL_TYPE, LETTER_TYPE, PRIORITY, KEY_TYPE_TEST, KEY_TYPE_TEAM
+from app.celery.tasks import build_dvla_file, update_job_to_sent_to_dvla
 from app.notifications.process_notifications import (
     persist_notification,
     send_notification_to_queue,
     simulated_recipient,
     persist_scheduled_notification)
+from app.notifications.process_letter_notifications import (
+    create_letter_api_job,
+    create_letter_notification
+)
 from app.notifications.validators import (
     validate_and_format_recipient,
     check_rate_limiting,
@@ -18,6 +24,7 @@ from app.notifications.validators import (
     validate_template
 )
 from app.schema_validation import validate
+from app.v2.errors import BadRequestError
 from app.v2.notifications import v2_notification_blueprint
 from app.v2.notifications.notification_schemas import (
     post_sms_request,
@@ -29,6 +36,7 @@ from app.v2.notifications.create_response import (
     create_post_email_response_from_notification,
     create_post_letter_response_from_notification
 )
+from app.variables import LETTER_TEST_API_FILENAME
 
 
 @v2_notification_blueprint.route('/<notification_type>', methods=['POST'])
@@ -58,10 +66,9 @@ def post_notification(notification_type):
 
     if notification_type == LETTER_TYPE:
         notification = process_letter_notification(
-            form=form,
+            letter_data=form,
             api_key=api_user,
             template=template,
-            service=authenticated_service,
         )
     else:
         notification = process_sms_or_email_notification(
@@ -140,10 +147,25 @@ def process_sms_or_email_notification(*, form, notification_type, api_key, templ
     return notification
 
 
-def process_letter_notification(*, form, api_key, template, service):
-    # create job
+def process_letter_notification(*, letter_data, api_key, template):
+    if api_key.key_type == KEY_TYPE_TEAM:
+        raise BadRequestError(message='Cannot send letters with a team api key', status_code=403)
 
-    # create notification
+    job = create_letter_api_job(template)
+    notification = create_letter_notification(letter_data, job, api_key)
 
-    # trigger build_dvla_file task
-    raise NotImplementedError
+    if api_key.service.research_mode or api_key.key_type == KEY_TYPE_TEST:
+
+        # distinguish real API jobs from test jobs by giving the test jobs a different filename
+        job.original_file_name = LETTER_TEST_API_FILENAME
+        dao_update_job(job)
+
+        update_job_to_sent_to_dvla.apply_async([str(job.id)], queue=QueueNames.RESEARCH_MODE)
+    else:
+        build_dvla_file.apply_async([str(job.id)], queue=QueueNames.JOBS)
+
+    current_app.logger.info("send job {} for api notification {} to build-dvla-file in the process-job queue".format(
+        job.id,
+        notification.id
+    ))
+    return notification

app/variables.py

@@ -0,0 +1,3 @@
+# all jobs for letters created via the api must have this filename
+LETTER_API_FILENAME = 'letter submitted via api'
+LETTER_TEST_API_FILENAME = 'test letter submitted via api'