Merge branch 'master' into secure-endpoints

Conflicts:
	app/__init__.py

app/celery/tasks.py

@@ -1,10 +1,14 @@
+import random
+
+import botocore
+from boto3 import resource
 from datetime import (datetime)
 
 from flask import current_app
 from notifications_utils.recipients import (
     RecipientCSV
 )
-from notifications_utils.template import SMSMessageTemplate, WithSubjectTemplate
+from notifications_utils.template import SMSMessageTemplate, WithSubjectTemplate, LetterDVLATemplate
 from sqlalchemy.exc import SQLAlchemyError
 from app import (
     create_uuid,
@@ -16,8 +20,9 @@ from app.aws import s3
 from app.celery import provider_tasks
 from app.dao.jobs_dao import (
     dao_update_job,
-    dao_get_job_by_id
-)
+    dao_get_job_by_id,
+    all_notifications_are_created_for_job,
+    dao_get_all_notifications_for_job)
 from app.dao.notifications_dao import get_notification_by_id
 from app.dao.services_dao import dao_fetch_service_by_id, fetch_todays_total_message_count
 from app.dao.templates_dao import dao_get_template_by_id
@@ -76,6 +81,8 @@ def process_job(job_id):
     current_app.logger.info(
         "Job {} created at {} started at {} finished at {}".format(job_id, job.created_at, start, finished)
     )
+    if template.template_type == LETTER_TYPE:
+        build_dvla_file.apply_async([str(job.id)], queue='process-job')
 
 
 def process_row(row_number, recipient, personalisation, template, job, service):
@@ -248,13 +255,62 @@ def persist_letter(
             notification_id=notification_id
         )
 
-        # TODO: deliver letters
-
         current_app.logger.info("Letter {} created at {}".format(saved_notification.id, created_at))
     except SQLAlchemyError as e:
         handle_exception(self, notification, notification_id, e)
 
 
+@notify_celery.task(bind=True, name="build-dvla-file", max_retries=15, default_retry_delay=300)
+@statsd(namespace="tasks")
+def build_dvla_file(self, job_id):
+    if all_notifications_are_created_for_job(job_id):
+        notifications = dao_get_all_notifications_for_job(job_id)
+        file = ""
+        for n in notifications:
+            t = {"content": n.template.content, "subject": n.template.subject}
+            # This unique id is a 7 digits requested by DVLA, not known if this number needs to be sequential.
+            unique_id = int(''.join(map(str, random.sample(range(9), 7))))
+            template = LetterDVLATemplate(t, n.personalisation, unique_id)
+            file = file + str(template) + "\n"
+        s3upload(filedata=file,
+                 region=current_app.config['AWS_REGION'],
+                 bucket_name=current_app.config['DVLA_UPLOAD_BUCKET_NAME'],
+                 file_location="{}-dvla-job.text".format(job_id))
+    else:
+        self.retry(queue="retry", exc="All notifications for job {} are not persisted".format(job_id))
+
+
+def s3upload(filedata, region, bucket_name, file_location):
+    # TODO: move this method to utils. Will need to change the filedata from here to send contents in filedata['data']
+    _s3 = resource('s3')
+    # contents = filedata['data']
+    contents = filedata
+
+    exists = True
+    try:
+        _s3.meta.client.head_bucket(
+            Bucket=bucket_name)
+    except botocore.exceptions.ClientError as e:
+        error_code = int(e.response['Error']['Code'])
+        if error_code == 404:
+            exists = False
+        else:
+            current_app.logger.error(
+                "Unable to create s3 bucket {}".format(bucket_name))
+            raise e
+
+    if not exists:
+        _s3.create_bucket(Bucket=bucket_name,
+                          CreateBucketConfiguration={'LocationConstraint': region})
+
+    upload_id = create_uuid()
+    upload_file_name = file_location
+    key = _s3.Object(bucket_name, upload_file_name)
+    key.put(Body=contents, ServerSideEncryption='AES256')
+
+    return upload_id
+
+
 def handle_exception(task, notification, notification_id, exc):
     if not get_notification_by_id(notification_id):
         retry_msg = '{task} notification for job {job} row number {row} and notification id {noti}'.format(

app/config.py

@@ -175,6 +175,8 @@ class Config(object):
     FUNCTIONAL_TEST_PROVIDER_SERVICE_ID = None
     FUNCTIONAL_TEST_PROVIDER_SMS_TEMPLATE_ID = None
 
+    DVLA_UPLOAD_BUCKET_NAME = "{}-dvla-file-per-job".format(os.getenv('NOTIFY_ENVIRONMENT'))
+
 
 ######################
 # Config overrides ###

app/dao/jobs_dao.py

@@ -5,7 +5,11 @@ from sqlalchemy import func, desc, asc, cast, Date as sql_date
 
 from app import db
 from app.dao import days_ago
-from app.models import Job, NotificationHistory, JOB_STATUS_SCHEDULED, JOB_STATUS_PENDING
+from app.models import (Job,
+                        Notification,
+                        NotificationHistory,
+                        JOB_STATUS_SCHEDULED,
+                        JOB_STATUS_PENDING)
 from app.statsd_decorators import statsd
 
 
@@ -24,6 +28,22 @@ def dao_get_notification_outcomes_for_job(service_id, job_id):
         .all()
 
 
+@statsd(namespace="dao")
+def all_notifications_are_created_for_job(job_id):
+    query = db.session.query(func.count(Notification.id), Job.id)\
+        .join(Job)\
+        .filter(Job.id == job_id)\
+        .group_by(Job.id)\
+        .having(func.count(Notification.id) == Job.notification_count).all()
+
+    return query
+
+
+@statsd(namespace="dao")
+def dao_get_all_notifications_for_job(job_id):
+    return db.session.query(Notification).filter(Notification.job_id == job_id).all()
+
+
 def dao_get_job_by_service_id_and_job_id(service_id, job_id):
     return Job.query.filter_by(service_id=service_id, id=job_id).one()
 

app/models.py

@@ -320,6 +320,21 @@ class Template(db.Model):
             _external=True
         )
 
+    def serialize(self):
+
+        serialized = {
+            "id": self.id,
+            "type": self.template_type,
+            "created_at": self.created_at.strftime(DATETIME_FORMAT),
+            "updated_at": self.updated_at.strftime(DATETIME_FORMAT) if self.updated_at else None,
+            "created_by": self.created_by.email_address,
+            "version": self.version,
+            "body": self.content,
+            "subject": self.subject if self.template_type == EMAIL_TYPE else None
+        }
+
+        return serialized
+
 
 class TemplateHistory(db.Model):
     __tablename__ = 'templates_history'
@@ -343,6 +358,21 @@ class TemplateHistory(db.Model):
                              nullable=False,
                              default=NORMAL)
 
+    def serialize(self):
+
+        serialized = {
+            "id": self.id,
+            "type": self.template_type,
+            "created_at": self.created_at.strftime(DATETIME_FORMAT),
+            "updated_at": self.updated_at.strftime(DATETIME_FORMAT) if self.updated_at else None,
+            "created_by": self.created_by.email_address,
+            "version": self.version,
+            "body": self.content,
+            "subject": self.subject if self.template_type == EMAIL_TYPE else None
+        }
+
+        return serialized
+
 
 MMG_PROVIDER = "mmg"
 FIRETEXT_PROVIDER = "firetext"

app/v2/template/__init__.py

@@ -0,0 +1,7 @@
+from flask import Blueprint
+
+from app.v2.errors import register_errors
+
+template_blueprint = Blueprint("v2_template", __name__, url_prefix='/v2/template')
+
+register_errors(template_blueprint)

app/v2/template/get_template.py

@@ -0,0 +1,30 @@
+import uuid
+
+from flask import jsonify, request
+from jsonschema.exceptions import ValidationError
+from werkzeug.exceptions import abort
+
+from app import api_user
+from app.dao import templates_dao
+from app.schema_validation import validate
+from app.v2.template import template_blueprint
+from app.v2.template.template_schemas import get_template_by_id_request
+
+
+@template_blueprint.route("/<template_id>", methods=['GET'])
+@template_blueprint.route("/<template_id>/version/<int:version>", methods=['GET'])
+def get_template_by_id(template_id, version=None):
+    try:
+        _data = {}
+        _data['id'] = template_id
+        if version:
+            _data['version'] = version
+
+        data = validate(_data, get_template_by_id_request)
+    except ValueError or AttributeError:
+        abort(404)
+
+    template = templates_dao.dao_get_template_by_id_and_service_id(
+        template_id, api_user.service_id, data.get('version'))
+
+    return jsonify(template.serialize()), 200

app/v2/template/template_schemas.py

@@ -0,0 +1,41 @@
+from app.models import TEMPLATE_TYPES
+from app.schema_validation.definitions import uuid
+
+
+get_template_by_id_request = {
+    "$schema": "http://json-schema.org/draft-04/schema#",
+    "description": "schema for parameters allowed when getting template by id",
+    "type": "object",
+    "properties": {
+        "id": uuid,
+        "version": {"type": ["integer", "null"], "minimum": 1}
+    },
+    "required": ["id"],
+    "additionalProperties": False,
+}
+
+get_template_by_id_response = {
+    "$schema": "http://json-schema.org/draft-04/schema#",
+    "description": "GET template by id schema response",
+    "type": "object",
+    "title": "reponse v2/template",
+    "properties": {
+        "id": uuid,
+        "type": {"enum": TEMPLATE_TYPES},
+        "created_at": {
+            "format": "date-time",
+            "type": "string",
+            "description": "Date+time created"
+        },
+        "updated_at": {
+            "format": "date-time",
+            "type": ["string", "null"],
+            "description": "Date+time updated"
+        },
+        "created_by": {"type": "string"},
+        "version": {"type": "integer"},
+        "body": {"type": "string"},
+        "subject": {"type": ["string", "null"]}
+    },
+    "required": ["id", "type", "created_at", "updated_at", "version", "created_by", "body"]
+}