Merge pull request #2943 from alphagov/international-postage

Set international postage and international flag for international letters

app/celery/letters_pdf_tasks.py

@@ -4,6 +4,7 @@ from base64 import urlsafe_b64encode
 
 from botocore.exceptions import ClientError as BotoClientError
 from flask import current_app
+from notifications_utils.postal_address import PostalAddress
 
 from notifications_utils.statsd_decorators import statsd
 from notifications_utils.letter_timings import LETTER_PROCESSING_DEADLINE
@@ -43,8 +44,8 @@ from app.models import (
     NOTIFICATION_VALIDATION_FAILED,
     NOTIFICATION_VIRUS_SCAN_FAILED,
     POSTAGE_TYPES,
-    RESOLVE_POSTAGE_FOR_FILE_NAME
-)
+    RESOLVE_POSTAGE_FOR_FILE_NAME,
+    INTERNATIONAL_POSTAGE_TYPES)
 from app.cronitor import cronitor
 
 
@@ -393,8 +394,16 @@ def process_virus_scan_error(filename):
 
 
 def update_letter_pdf_status(reference, status, billable_units, recipient_address=None):
-
+    postage = None
+    if recipient_address:
+        # fix allow_international_letters
+        postage = PostalAddress(raw_address=recipient_address.replace(',', '\n'),
+                                allow_international_letters=True
+                                ).postage
+        postage = postage if postage in INTERNATIONAL_POSTAGE_TYPES else None
     update_dict = {'status': status, 'billable_units': billable_units, 'updated_at': datetime.utcnow()}
+    if postage:
+        update_dict.update({'postage': postage, 'international': True})
     if recipient_address:
         update_dict['to'] = recipient_address
     return dao_update_notifications_by_reference(

app/celery/tasks.py

@@ -354,7 +354,7 @@ def save_letter(
         saved_notification = persist_notification(
             template_id=notification['template'],
             template_version=notification['template_version'],
-            template_postage=template.postage,
+            postage=template.postage,
             recipient=recipient,
             service=service,
             personalisation=notification['personalisation'],

app/notifications/process_letter_notifications.py

@@ -14,11 +14,11 @@ def create_letter_notification(
     reply_to_text=None,
     billable_units=None,
     updated_at=None,
+    postage=None
 ):
     notification = persist_notification(
         template_id=template.id,
         template_version=template.version,
-        template_postage=template.postage,
         # we only accept addresses_with_underscores from the API (from CSV we also accept dashes, spaces etc)
         recipient=PostalAddress.from_personalisation(letter_data['personalisation']).normalised,
         service=service,
@@ -33,7 +33,9 @@ def create_letter_notification(
         status=status,
         reply_to_text=reply_to_text,
         billable_units=billable_units,
-        postage=letter_data.get('postage'),
+        # letter_data.get('postage') is only set for precompiled letters (if international it is set after sanitise)
+        # letters from a template will pass in 'europe' or 'rest-of-world' if None then use postage from template
+        postage=postage or letter_data.get('postage') or template.postage,
         updated_at=updated_at
     )
     return notification

app/notifications/process_notifications.py

@@ -27,7 +27,7 @@ from app.models import (
     LETTER_TYPE,
     NOTIFICATION_CREATED,
     Notification,
-)
+    INTERNATIONAL_POSTAGE_TYPES)
 from app.dao.notifications_dao import (
     dao_create_notification,
     dao_delete_notifications_by_id,
@@ -107,7 +107,6 @@ def persist_notification(
     reply_to_text=None,
     billable_units=None,
     postage=None,
-    template_postage=None,
     document_download_count=None,
     updated_at=None
 ):
@@ -147,7 +146,8 @@ def persist_notification(
     elif notification_type == EMAIL_TYPE:
         notification.normalised_to = format_email_address(notification.to)
     elif notification_type == LETTER_TYPE:
-        notification.postage = postage or template_postage
+        notification.postage = postage
+        notification.international = postage in INTERNATIONAL_POSTAGE_TYPES
         notification.normalised_to = ''.join(notification.to.split()).lower()
 
     # if simulated create a Notification model to return but do not persist the Notification to the dB

app/notifications/rest.py

@@ -120,7 +120,7 @@ def send_notification(notification_type):
     simulated = simulated_recipient(notification_form['to'], notification_type)
     notification_model = persist_notification(template_id=template.id,
                                               template_version=template.version,
-                                              template_postage=template.postage,
+                                              postage=template.postage,
                                               recipient=request.get_json()['to'],
                                               service=authenticated_service,
                                               personalisation=notification_form.get('personalisation', None),

app/notifications/validators.py

@@ -1,3 +1,4 @@
+from notifications_utils.postal_address import PostalAddress
 from sqlalchemy.orm.exc import NoResultFound
 from flask import current_app
 from notifications_utils import SMS_CHAR_COUNT_LIMIT
@@ -14,9 +15,9 @@ from app.models import (
     INTERNATIONAL_SMS_TYPE, SMS_TYPE, EMAIL_TYPE, LETTER_TYPE,
     KEY_TYPE_TEST, KEY_TYPE_TEAM,
     ServicePermission,
-)
+    INTERNATIONAL_LETTERS)
 from app.service.utils import service_allowed_to_send_to
-from app.v2.errors import TooManyRequestsError, BadRequestError, RateLimitError
+from app.v2.errors import TooManyRequestsError, BadRequestError, RateLimitError, ValidationError
 from app import redis_store
 from app.notifications.process_notifications import create_content_for_notification
 from app.utils import get_public_notify_type_text
@@ -214,3 +215,34 @@ def check_service_letter_contact_id(service_id, letter_contact_id, notification_
             message = 'letter_contact_id {} does not exist in database for service id {}'\
                 .format(letter_contact_id, service_id)
             raise BadRequestError(message=message)
+
+
+def validate_address(service, letter_data):
+    address = PostalAddress.from_personalisation(
+        letter_data,
+        allow_international_letters=(INTERNATIONAL_LETTERS in str(service.permissions)),
+    )
+    if not address.has_enough_lines:
+        raise ValidationError(
+            message=f'Address must be at least {PostalAddress.MIN_LINES} lines'
+        )
+    if address.has_too_many_lines:
+        raise ValidationError(
+            message=f'Address must be no more than {PostalAddress.MAX_LINES} lines'
+        )
+    if not address.has_valid_last_line:
+        if address.allow_international_letters:
+            raise ValidationError(
+                message=f'Last line of address must be a real UK postcode or another country'
+            )
+        raise ValidationError(
+            message='Must be a real UK postcode'
+        )
+    if address.has_invalid_characters:
+        raise ValidationError(
+            message='Address lines must not start with any of the following characters: @ ( ) = [ ] ” \\ / ,'
+        )
+    if address.postage == 'united-kingdom':
+        return None  # use postage from template
+    else:
+        return address.postage

app/service/send_notification.py

@@ -13,8 +13,8 @@ from app.notifications.validators import (
     check_service_has_permission,
     check_service_over_daily_message_limit,
     validate_and_format_recipient,
-    validate_template
-)
+    validate_template,
+    validate_address)
 from app.notifications.process_notifications import (
     persist_notification,
     send_notification_to_queue
@@ -75,7 +75,13 @@ def send_one_off_notification(service_id, post_data):
         notification_type=template.template_type,
         allow_guest_list_recipients=False,
     )
-
+    postage = None
+    if template.template_type == LETTER_TYPE:
+        # Validate address and set postage to europe|rest-of-world if international letter,
+        # otherwise persist_notification with use template postage
+        postage = validate_address(service, personalisation)
+        if not postage:
+            postage = template.postage
     validate_created_by(service, post_data['created_by'])
 
     sender_id = post_data.get('sender_id', None)
@@ -88,7 +94,6 @@ def send_one_off_notification(service_id, post_data):
     notification = persist_notification(
         template_id=template.id,
         template_version=template.version,
-        template_postage=template.postage,
         recipient=post_data['to'],
         service=service,
         personalisation=personalisation,
@@ -98,6 +103,7 @@ def send_one_off_notification(service_id, post_data):
         created_by_id=post_data['created_by'],
         reply_to_text=reply_to,
         reference=create_one_off_reference(template.template_type),
+        postage=postage
     )
 
     queue_name = QueueNames.PRIORITY if template.process_type == PRIORITY else None
@@ -173,7 +179,6 @@ def send_pdf_letter_notification(service_id, post_data):
         notification_id=post_data['file_id'],
         template_id=template.id,
         template_version=template.version,
-        template_postage=template.postage,
         recipient=urllib.parse.unquote(post_data['recipient_address']),
         service=service,
         personalisation=personalisation,
@@ -184,7 +189,7 @@ def send_pdf_letter_notification(service_id, post_data):
         client_reference=post_data['filename'],
         created_by_id=post_data['created_by'],
         billable_units=billable_units,
-        postage=post_data['postage'],
+        postage=post_data['postage'] or template.postage,
     )
 
     upload_filename = get_letter_pdf_filename(

app/v2/notifications/post_notifications.py

@@ -5,7 +5,6 @@ from datetime import datetime
 
 from boto.exception import SQSError
 from flask import request, jsonify, current_app, abort
-from notifications_utils.postal_address import PostalAddress
 from notifications_utils.recipients import try_validate_and_format_phone_number
 from gds_metrics import Histogram
 
@@ -36,7 +35,6 @@ from app.models import (
     NOTIFICATION_SENDING,
     NOTIFICATION_DELIVERED,
     NOTIFICATION_PENDING_VIRUS_CHECK,
-    INTERNATIONAL_LETTERS,
     Notification)
 from app.notifications.process_letter_notifications import (
     create_letter_notification
@@ -51,15 +49,18 @@ from app.notifications.validators import (
     check_service_email_reply_to_id,
     check_service_has_permission,
     check_service_sms_sender_id,
+    validate_address,
     validate_and_format_recipient,
     validate_template,
 )
 from app.schema_validation import validate
-from app.v2.errors import BadRequestError, ValidationError
-from app.v2.notifications import v2_notification_blueprint
+from app.v2.errors import BadRequestError
 from app.v2.notifications.create_response import (
-    create_post_sms_response_from_notification, create_post_email_response_from_notification,
-    create_post_letter_response_from_notification)
+    create_post_email_response_from_notification,
+    create_post_sms_response_from_notification,
+    create_post_letter_response_from_notification
+)
+from app.v2.notifications import v2_notification_blueprint
 from app.v2.notifications.notification_schemas import (
     post_sms_request,
     post_email_request,
@@ -339,7 +340,7 @@ def process_letter_notification(
                                                         template=template,
                                                         reply_to_text=reply_to_text)
 
-    validate_address(service, letter_data)
+    postage = validate_address(service, letter_data['personalisation'])
 
     test_key = api_key.key_type == KEY_TYPE_TEST
 
@@ -362,7 +363,8 @@ def process_letter_notification(
                                               api_key=api_key,
                                               status=status,
                                               reply_to_text=reply_to_text,
-                                              updated_at=updated_at
+                                              updated_at=updated_at,
+                                              postage=postage
                                               )
 
     get_pdf_for_templated_letter.apply_async(
@@ -388,33 +390,6 @@ def process_letter_notification(
     return resp
 
 
-def validate_address(service, letter_data):
-    address = PostalAddress.from_personalisation(
-        letter_data['personalisation'],
-        allow_international_letters=(INTERNATIONAL_LETTERS in service.permissions),
-    )
-    if not address.has_enough_lines:
-        raise ValidationError(
-            message=f'Address must be at least {PostalAddress.MIN_LINES} lines'
-        )
-    if address.has_too_many_lines:
-        raise ValidationError(
-            message=f'Address must be no more than {PostalAddress.MAX_LINES} lines'
-        )
-    if not address.has_valid_last_line:
-        if address.allow_international_letters:
-            raise ValidationError(
-                message=f'Last line of address must be a real UK postcode or another country'
-            )
-        raise ValidationError(
-            message='Must be a real UK postcode'
-        )
-    if address.has_invalid_characters:
-        raise ValidationError(
-            message='Address lines must not start with any of the following characters: @ ( ) = [ ] ” \\ / ,'
-        )
-
-
 def process_precompiled_letter_notifications(*, letter_data, api_key, service, template, reply_to_text):
     try:
         status = NOTIFICATION_PENDING_VIRUS_CHECK