Replace placeholders before sending a message

This commit replaces placeholders in a template with the user’s data, using
the Template class from utils
(https://github.com/alphagov/notifications-utils/tree/master/utils#template)

It also extracts the personalisation data from the CSV, taking account of the
different column headings that SMS and email CSVs will have.

At the point of creating the task to send an individual messages, validation of
the placeholders matching the template is assumed to have been done either:
- in processing the CSV in the admin app
- in the endpoint for the API call

No exceptions should be raised at this point.

app/celery/tasks.py

@@ -9,8 +9,9 @@ from app.models import Notification
 from flask import current_app
 from sqlalchemy.exc import SQLAlchemyError
 from app.aws import s3
-from app.csv import get_recipient_from_csv
 from datetime import datetime
+from utils.template import Template
+from utils.process_csv import get_rows_from_csv, get_recipient_from_row, first_column_heading
 
 
 @notify_celery.task(name="process-job")
@@ -21,13 +22,14 @@ def process_job(job_id):
     dao_update_job(job)
 
     file = s3.get_job_from_s3(job.bucket_name, job_id)
-    recipients = get_recipient_from_csv(file)
 
-    for recipient in recipients:
+    for row in get_rows_from_csv(file):
+
         encrypted = encryption.encrypt({
             'template': job.template_id,
             'job': str(job.id),
-            'to': recipient
+            'to': get_recipient_from_row(row, job.template.template_type),
+            'personalisation': row
         })
 
         if job.template.template_type == 'sms':
@@ -62,7 +64,10 @@ def process_job(job_id):
 @notify_celery.task(name="send-sms")
 def send_sms(service_id, notification_id, encrypted_notification, created_at):
     notification = encryption.decrypt(encrypted_notification)
-    template = dao_get_template_by_id(notification['template'])
+    template = Template(
+        dao_get_template_by_id(notification['template']),
+        values=notification.get('personalisation', {})
+    )
 
     client = firetext_client
 
@@ -78,12 +83,14 @@ def send_sms(service_id, notification_id, encrypted_notification, created_at):
             created_at=created_at,
             sent_at=sent_at,
             sent_by=client.get_name()
-
         )
         dao_create_notification(notification_db_object)
 
         try:
-            client.send_sms(notification['to'], template.content)
+            client.send_sms(
+                notification['to'],
+                template.replaced
+            )
         except FiretextClientException as e:
             current_app.logger.debug(e)
             notification_db_object.status = 'failed'
@@ -99,7 +106,10 @@ def send_sms(service_id, notification_id, encrypted_notification, created_at):
 @notify_celery.task(name="send-email")
 def send_email(service_id, notification_id, subject, from_address, encrypted_notification, created_at):
     notification = encryption.decrypt(encrypted_notification)
-    template = dao_get_template_by_id(notification['template'])
+    template = Template(
+        dao_get_template_by_id(notification['template']),
+        values=notification.get('personalisation', {})
+    )
 
     client = aws_ses_client
 
@@ -123,7 +133,7 @@ def send_email(service_id, notification_id, subject, from_address, encrypted_not
                 from_address,
                 notification['to'],
                 subject,
-                template.content
+                template.replaced
             )
         except AwsSesClientException as e:
             current_app.logger.debug(e)