Merge branch 'master' into update_template_schema

app/celery/letters_pdf_tasks.py

@@ -98,7 +98,8 @@ def collate_letter_pdfs_for_day(date):
         notify_celery.send_task(
             name=TaskNames.ZIP_AND_SEND_LETTER_PDFS,
             kwargs={'filenames_to_zip': filenames},
-            queue=QueueNames.PROCESS_FTP
+            queue=QueueNames.PROCESS_FTP,
+            compression='zlib'
         )
 
 
@@ -112,7 +113,10 @@ def group_letters(letter_pdfs):
     list_of_files = []
     for letter in letter_pdfs:
         if letter['Key'].lower().endswith('.pdf'):
-            if running_filesize + letter['Size'] > current_app.config['MAX_LETTER_PDF_ZIP_FILESIZE']:
+            if (
+                running_filesize + letter['Size'] > current_app.config['MAX_LETTER_PDF_ZIP_FILESIZE'] or
+                len(list_of_files) >= current_app.config['MAX_LETTER_PDF_COUNT_PER_ZIP']
+            ):
                 yield list_of_files
                 running_filesize = 0
                 list_of_files = []

app/celery/tasks.py

@@ -31,7 +31,6 @@ from app import (
 from app.aws import s3
 from app.celery import provider_tasks
 from app.celery import letters_pdf_tasks
-from app.celery.service_callback_tasks import send_delivery_status_to_service
 from app.config import QueueNames
 from app.dao.inbound_sms_dao import dao_get_inbound_sms_by_id
 from app.dao.jobs_dao import (
@@ -46,13 +45,11 @@ from app.dao.notifications_dao import (
     dao_update_notifications_for_job_to_sent_to_dvla,
     dao_update_notifications_by_reference,
     dao_get_last_notification_added_for_job_id,
-    dao_get_notifications_by_references,
 )
 from app.dao.provider_details_dao import get_current_provider
 from app.dao.service_inbound_api_dao import get_service_inbound_api_for_service
 from app.dao.services_dao import dao_fetch_service_by_id, fetch_todays_total_message_count
 from app.dao.templates_dao import dao_get_template_by_id
-from app.dao.service_callback_api_dao import get_service_callback_api_for_service
 from app.models import (
     DVLA_RESPONSE_STATUS_SENT,
     EMAIL_TYPE,
@@ -416,12 +413,6 @@ def update_letter_notifications_to_error(self, notification_references):
     )
 
     current_app.logger.info("Updated {} letter notifications to technical-failure".format(updated_count))
-    notifications = dao_get_notifications_by_references(references=notification_references)
-    # queue callback task only if the service_callback_api exists
-    service_callback_api = get_service_callback_api_for_service(service_id=notifications[0].service_id)
-    if service_callback_api:
-        for notification in notifications:
-            send_delivery_status_to_service.apply_async([str(notification.id)], queue=QueueNames.CALLBACKS)
 
 
 def create_dvla_file_contents_for_job(job_id):
@@ -503,12 +494,6 @@ def update_letter_notifications_statuses(self, filename):
                 current_app.logger.info(
                     'DVLA file: {filename}, notification updated to {status}: {reference}'.format(
                         filename=filename, status=status, reference=str(update.reference)))
-                notifications = dao_get_notifications_by_references(references=[update.reference])
-                # queue callback task only if the service_callback_api exists
-                service_callback_api = get_service_callback_api_for_service(service_id=notifications[0].service_id)
-                if service_callback_api:
-                    for notification in notifications:
-                        send_delivery_status_to_service.apply_async([str(notification.id)], queue=QueueNames.CALLBACKS)
 
 
 def process_updates_from_file(response_file):

app/config.py

@@ -129,6 +129,7 @@ class Config(object):
     MAX_VERIFY_CODE_COUNT = 10
 
     MAX_LETTER_PDF_ZIP_FILESIZE = 500 * 1024 * 1024  # 500mb
+    MAX_LETTER_PDF_COUNT_PER_ZIP = 5000
 
     CHECK_PROXY_HEADER = False
 

app/invite/rest.py

@@ -23,7 +23,8 @@ register_errors(invite)
 
 @invite.route('', methods=['POST'])
 def create_invited_user(service_id):
-    invited_user, errors = invited_user_schema.load(request.get_json())
+    request_json = request.get_json()
+    invited_user, errors = invited_user_schema.load(request_json)
     save_invited_user(invited_user)
 
     template = dao_get_template_by_id(current_app.config['INVITATION_EMAIL_TEMPLATE_ID'])
@@ -37,7 +38,10 @@ def create_invited_user(service_id):
         personalisation={
             'user_name': invited_user.from_user.name,
             'service_name': invited_user.service.name,
-            'url': invited_user_url(invited_user.id)
+            'url': invited_user_url(
+                invited_user.id,
+                request_json.get('invite_link_host'),
+            ),
         },
         notification_type=EMAIL_TYPE,
         api_key_id=None,
@@ -74,8 +78,11 @@ def update_invited_user(service_id, invited_user_id):
     return jsonify(data=invited_user_schema.dump(fetched).data), 200
 
 
-def invited_user_url(invited_user_id):
+def invited_user_url(invited_user_id, invite_link_host=None):
     from notifications_utils.url_safe_token import generate_token
     token = generate_token(str(invited_user_id), current_app.config['SECRET_KEY'], current_app.config['DANGEROUS_SALT'])
 
-    return '{0}/invitation/{1}'.format(current_app.config['ADMIN_BASE_URL'], token)
+    if invite_link_host is None:
+        invite_link_host = current_app.config['ADMIN_BASE_URL']
+
+    return '{0}/invitation/{1}'.format(invite_link_host, token)

app/models.py

@@ -14,6 +14,7 @@ from sqlalchemy import UniqueConstraint, CheckConstraint
 from notifications_utils.recipients import (
     validate_email_address,
     validate_phone_number,
+    try_validate_and_format_phone_number,
     InvalidPhoneError,
     InvalidEmailError
 )
@@ -366,6 +367,9 @@ class ServiceSmsSender(db.Model):
     created_at = db.Column(db.DateTime, default=datetime.datetime.utcnow, nullable=False)
     updated_at = db.Column(db.DateTime, nullable=True, onupdate=datetime.datetime.utcnow)
 
+    def get_reply_to_text(self):
+        return try_validate_and_format_phone_number(self.sms_sender)
+
     def serialize(self):
         return {
             "id": str(self.id),
@@ -640,7 +644,7 @@ class TemplateBase(db.Model):
         elif self.template_type == EMAIL_TYPE:
             return self.service.get_default_reply_to_email_address()
         elif self.template_type == SMS_TYPE:
-            return self.service.get_default_sms_sender()
+            return try_validate_and_format_phone_number(self.service.get_default_sms_sender())
         else:
             return None
 

app/service/send_notification.py

@@ -90,7 +90,7 @@ def get_reply_to_text(notification_type, sender_id, service, template):
         if notification_type == EMAIL_TYPE:
             reply_to = dao_get_reply_to_by_id(service.id, sender_id).email_address
         elif notification_type == SMS_TYPE:
-            reply_to = dao_get_service_sms_senders_by_id(service.id, sender_id).sms_sender
+            reply_to = dao_get_service_sms_senders_by_id(service.id, sender_id).get_reply_to_text()
     else:
         reply_to = template.get_reply_to_text()
 

app/v2/notifications/post_notifications.py

@@ -2,6 +2,8 @@ import functools
 
 from flask import request, jsonify, current_app, abort
 
+from notifications_utils.recipients import try_validate_and_format_phone_number
+
 from app import api_user, authenticated_service
 from app.config import QueueNames
 from app.models import (
@@ -208,9 +210,13 @@ def get_reply_to_text(notification_type, form, template):
 
     elif notification_type == SMS_TYPE:
         service_sms_sender_id = form.get("sms_sender_id", None)
-        reply_to = check_service_sms_sender_id(
+        sms_sender_id = check_service_sms_sender_id(
             str(authenticated_service.id), service_sms_sender_id, notification_type
-        ) or template.get_reply_to_text()
+        )
+        if sms_sender_id:
+            reply_to = try_validate_and_format_phone_number(sms_sender_id)
+        else:
+            reply_to = template.get_reply_to_text()
 
     elif notification_type == LETTER_TYPE:
         reply_to = template.get_reply_to_text()