Remove letters-related code (#175)

This deletes a big ol' chunk of code related to letters. It's not everything—there are still a few things that might be tied to sms/email—but it's the the heart of letters function. SMS and email function should be untouched by this.

Areas affected:

- Things obviously about letters
- PDF tasks, used for precompiling letters
- Virus scanning, used for those PDFs
- FTP, used to send letters to the printer
- Postage stuff

app/service/rest.py

@@ -2,10 +2,6 @@ import itertools
 from datetime import datetime
 
 from flask import Blueprint, current_app, jsonify, request
-from notifications_utils.letter_timings import (
-    letter_can_be_cancelled,
-    too_late_to_cancel_letter,
-)
 from notifications_utils.timezones import convert_utc_to_local_timezone
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm.exc import NoResultFound
@@ -32,12 +28,6 @@ from app.dao.fact_notification_status_dao import (
 )
 from app.dao.inbound_numbers_dao import dao_allocate_number_for_service
 from app.dao.organisation_dao import dao_get_organisation_by_service_id
-from app.dao.returned_letters_dao import (
-    fetch_most_recent_returned_letter,
-    fetch_recent_returned_letter_count,
-    fetch_returned_letter_summary,
-    fetch_returned_letters,
-)
 from app.dao.service_contact_list_dao import (
     dao_archive_contact_list,
     dao_get_contact_list_by_id,
@@ -63,13 +53,6 @@ from app.dao.service_guest_list_dao import (
     dao_fetch_service_guest_list,
     dao_remove_service_guest_list,
 )
-from app.dao.service_letter_contact_dao import (
-    add_letter_contact_for_service,
-    archive_letter_contact,
-    dao_get_letter_contact_by_id,
-    dao_get_letter_contacts_by_service_id,
-    update_letter_contact,
-)
 from app.dao.service_sms_sender_dao import (
     archive_sms_sender,
     dao_add_sms_sender_for_service,
@@ -97,13 +80,9 @@ from app.dao.services_dao import (
 from app.dao.templates_dao import dao_get_template_by_id
 from app.dao.users_dao import get_user_by_id
 from app.errors import InvalidRequest, register_errors
-from app.letters.utils import letter_print_day
 from app.models import (
     KEY_TYPE_NORMAL,
-    LETTER_TYPE,
-    NOTIFICATION_CANCELLED,
     EmailBranding,
-    LetterBranding,
     Permission,
     Service,
     ServiceContactList,
@@ -122,11 +101,7 @@ from app.schemas import (
     service_schema,
 )
 from app.service import statistics
-from app.service.send_notification import (
-    send_one_off_notification,
-    send_pdf_letter_notification,
-)
-from app.service.send_pdf_letter_schema import send_pdf_letter_request
+from app.service.send_notification import send_one_off_notification
 from app.service.sender import send_notification_to_service_users
 from app.service.service_contact_list_schema import (
     create_service_contact_list_schema,
@@ -137,17 +112,11 @@ from app.service.service_data_retention_schema import (
 )
 from app.service.service_senders_schema import (
     add_service_email_reply_to_request,
-    add_service_letter_contact_block_request,
     add_service_sms_sender_request,
 )
 from app.service.utils import get_guest_list_objects
 from app.user.users_schema import post_set_permissions_schema
-from app.utils import (
-    DATE_FORMAT,
-    DATETIME_FORMAT_NO_TIMEZONE,
-    get_prev_next_pagination_links,
-    midnight_n_days_ago,
-)
+from app.utils import get_prev_next_pagination_links
 
 service_blueprint = Blueprint('service', __name__)
 
@@ -275,9 +244,6 @@ def update_service(service_id):
     if 'email_branding' in req_json:
         email_branding_id = req_json['email_branding']
         service.email_branding = None if not email_branding_id else EmailBranding.query.get(email_branding_id)
-    if 'letter_branding' in req_json:
-        letter_branding_id = req_json['letter_branding']
-        service.letter_branding = None if not letter_branding_id else LetterBranding.query.get(letter_branding_id)
     dao_update_service(service)
 
     if service_going_live:
@@ -491,39 +457,6 @@ def get_notification_for_service(service_id, notification_id):
     ), 200
 
 
-@service_blueprint.route('/<uuid:service_id>/notifications/<uuid:notification_id>/cancel', methods=['POST'])
-def cancel_notification_for_service(service_id, notification_id):
-    notification = notifications_dao.get_notification_by_id(notification_id, service_id)
-
-    if not notification:
-        raise InvalidRequest('Notification not found', status_code=404)
-    elif notification.notification_type != LETTER_TYPE:
-        raise InvalidRequest('Notification cannot be cancelled - only letters can be cancelled', status_code=400)
-    elif not letter_can_be_cancelled(notification.status, notification.created_at):
-        print_day = letter_print_day(notification.created_at)
-        if too_late_to_cancel_letter(notification.created_at):
-            message = "It’s too late to cancel this letter. Printing started {} at 5.30pm".format(print_day)
-        elif notification.status == 'cancelled':
-            message = "This letter has already been cancelled."
-        else:
-            message = (
-                f"We could not cancel this letter. "
-                f"Letter status: {notification.status}, created_at: {notification.created_at}"
-            )
-        raise InvalidRequest(
-            message,
-            status_code=400)
-
-    updated_notification = notifications_dao.update_notification_status_by_id(
-        notification_id,
-        NOTIFICATION_CANCELLED,
-    )
-
-    return jsonify(
-        notification_with_template_schema.dump(updated_notification)
-    ), 200
-
-
 def search_for_notification_by_to_field(service_id, search_term, statuses, notification_type):
     results = notifications_dao.dao_get_notifications_by_recipient_or_reference(
         service_id=service_id,
@@ -734,7 +667,6 @@ def get_monthly_template_usage(service_id):
                     'month': i.month,
                     'year': i.year,
                     'count': i.count,
-                    'is_precompiled_letter': i.is_precompiled_letter
                 }
             )
 
@@ -749,13 +681,6 @@ def create_one_off_notification(service_id):
     return jsonify(resp), 201
 
 
-@service_blueprint.route('/<uuid:service_id>/send-pdf-letter', methods=['POST'])
-def create_pdf_letter(service_id):
-    data = validate(request.get_json(), send_pdf_letter_request)
-    resp = send_pdf_letter_notification(service_id, data)
-    return jsonify(resp), 201
-
-
 @service_blueprint.route('/<uuid:service_id>/email-reply-to', methods=["GET"])
 def get_email_reply_to_addresses(service_id):
     result = dao_get_reply_to_by_service_id(service_id)
@@ -823,48 +748,6 @@ def delete_service_reply_to_email_address(service_id, reply_to_email_id):
     return jsonify(data=archived_reply_to.serialize()), 200
 
 
-@service_blueprint.route('/<uuid:service_id>/letter-contact', methods=["GET"])
-def get_letter_contacts(service_id):
-    result = dao_get_letter_contacts_by_service_id(service_id)
-    return jsonify([i.serialize() for i in result]), 200
-
-
-@service_blueprint.route('/<uuid:service_id>/letter-contact/<uuid:letter_contact_id>', methods=["GET"])
-def get_letter_contact_by_id(service_id, letter_contact_id):
-    result = dao_get_letter_contact_by_id(service_id=service_id, letter_contact_id=letter_contact_id)
-    return jsonify(result.serialize()), 200
-
-
-@service_blueprint.route('/<uuid:service_id>/letter-contact', methods=['POST'])
-def add_service_letter_contact(service_id):
-    # validate the service exists, throws ResultNotFound exception.
-    dao_fetch_service_by_id(service_id)
-    form = validate(request.get_json(), add_service_letter_contact_block_request)
-    new_letter_contact = add_letter_contact_for_service(service_id=service_id,
-                                                        contact_block=form['contact_block'],
-                                                        is_default=form.get('is_default', True))
-    return jsonify(data=new_letter_contact.serialize()), 201
-
-
-@service_blueprint.route('/<uuid:service_id>/letter-contact/<uuid:letter_contact_id>', methods=['POST'])
-def update_service_letter_contact(service_id, letter_contact_id):
-    # validate the service exists, throws ResultNotFound exception.
-    dao_fetch_service_by_id(service_id)
-    form = validate(request.get_json(), add_service_letter_contact_block_request)
-    new_reply_to = update_letter_contact(service_id=service_id,
-                                         letter_contact_id=letter_contact_id,
-                                         contact_block=form['contact_block'],
-                                         is_default=form.get('is_default', True))
-    return jsonify(data=new_reply_to.serialize()), 200
-
-
-@service_blueprint.route('/<uuid:service_id>/letter-contact/<uuid:letter_contact_id>/archive', methods=['POST'])
-def delete_service_letter_contact(service_id, letter_contact_id):
-    archived_letter_contact = archive_letter_contact(service_id, letter_contact_id)
-
-    return jsonify(data=archived_letter_contact.serialize()), 200
-
-
 @service_blueprint.route('/<uuid:service_id>/sms-sender', methods=['POST'])
 def add_service_sms_sender(service_id):
     dao_fetch_service_by_id(service_id)
@@ -1042,71 +925,6 @@ def check_if_reply_to_address_already_in_use(service_id, email_address):
         )
 
 
-@service_blueprint.route('/<uuid:service_id>/returned-letter-statistics', methods=['GET'])
-def returned_letter_statistics(service_id):
-
-    most_recent = fetch_most_recent_returned_letter(service_id)
-
-    if not most_recent:
-        return jsonify({
-            'returned_letter_count': 0,
-            'most_recent_report': None,
-        })
-
-    most_recent_reported_at = datetime.combine(
-        most_recent.reported_at, datetime.min.time()
-    )
-
-    if most_recent_reported_at < midnight_n_days_ago(7):
-        return jsonify({
-            'returned_letter_count': 0,
-            'most_recent_report': most_recent.reported_at.strftime(DATETIME_FORMAT_NO_TIMEZONE),
-        })
-
-    count = fetch_recent_returned_letter_count(service_id)
-
-    return jsonify({
-        'returned_letter_count': count.returned_letter_count,
-        'most_recent_report': most_recent.reported_at.strftime(DATETIME_FORMAT_NO_TIMEZONE),
-    })
-
-
-@service_blueprint.route('/<uuid:service_id>/returned-letter-summary', methods=['GET'])
-def returned_letter_summary(service_id):
-    results = fetch_returned_letter_summary(service_id)
-
-    json_results = [{'returned_letter_count': x.returned_letter_count,
-                     'reported_at': x.reported_at.strftime(DATE_FORMAT)
-                     } for x in results]
-
-    return jsonify(json_results)
-
-
-@service_blueprint.route('/<uuid:service_id>/returned-letters', methods=['GET'])
-def get_returned_letters(service_id):
-    results = fetch_returned_letters(service_id=service_id, report_date=request.args.get('reported_at'))
-
-    json_results = [
-        {'notification_id': x.notification_id,
-         # client reference can only be added on API letters
-         'client_reference': x.client_reference if x.api_key_id else None,
-         'reported_at': x.reported_at.strftime(DATE_FORMAT),
-         'created_at': x.created_at.strftime(DATETIME_FORMAT_NO_TIMEZONE),
-         # it doesn't make sense to show hidden/precompiled templates
-         'template_name': x.template_name if not x.hidden else None,
-         'template_id': x.template_id if not x.hidden else None,
-         'template_version': x.template_version if not x.hidden else None,
-         'user_name': x.user_name or 'API',
-         'email_address': x.email_address or 'API',
-         'original_file_name': x.original_file_name,
-         'job_row_number': x.job_row_number,
-         # the file name for a letter uploaded via the UI
-         'uploaded_letter_file_name': x.client_reference if x.hidden and not x.api_key_id else None
-         } for x in results]
-
-    return jsonify(sorted(json_results, key=lambda i: i['created_at'], reverse=True))
-
-
 @service_blueprint.route('/<uuid:service_id>/contact-list', methods=['GET'])
 def get_contact_list(service_id):
     contact_lists = dao_get_contact_lists(service_id)

app/service/send_notification.py

@@ -1,46 +1,18 @@
-import urllib
-
-from flask import current_app
-from notifications_utils.s3 import S3ObjectNotFound
-from notifications_utils.s3 import s3download as utils_s3download
 from sqlalchemy.orm.exc import NoResultFound
 
-from app import create_random_identifier
 from app.config import QueueNames
-from app.dao.notifications_dao import (
-    _update_notification_status,
-    get_notification_by_id,
-)
 from app.dao.service_email_reply_to_dao import dao_get_reply_to_by_id
 from app.dao.service_sms_sender_dao import dao_get_service_sms_senders_by_id
 from app.dao.services_dao import dao_fetch_service_by_id
-from app.dao.templates_dao import (
-    dao_get_template_by_id_and_service_id,
-    get_precompiled_letter_template,
-)
+from app.dao.templates_dao import dao_get_template_by_id_and_service_id
 from app.dao.users_dao import get_user_by_id
-from app.letters.utils import (
-    generate_letter_pdf_filename,
-    get_billable_units_for_letter_page_count,
-    get_page_count,
-    move_uploaded_pdf_to_letters_bucket,
-)
-from app.models import (
-    EMAIL_TYPE,
-    KEY_TYPE_NORMAL,
-    LETTER_TYPE,
-    NOTIFICATION_DELIVERED,
-    PRIORITY,
-    SMS_TYPE,
-)
+from app.models import EMAIL_TYPE, KEY_TYPE_NORMAL, PRIORITY, SMS_TYPE
 from app.notifications.process_notifications import (
     persist_notification,
     send_notification_to_queue,
 )
 from app.notifications.validators import (
-    check_service_has_permission,
     check_service_over_daily_message_limit,
-    validate_address,
     validate_and_format_recipient,
     validate_template,
 )
@@ -57,9 +29,8 @@ def validate_created_by(service, created_by_id):
         raise BadRequestError(message=message)
 
 
+# TODO: possibly unnecessary after removing letters
 def create_one_off_reference(template_type):
-    if template_type == LETTER_TYPE:
-        return create_random_identifier()
     return None
 
 
@@ -83,16 +54,7 @@ def send_one_off_notification(service_id, post_data):
         notification_type=template.template_type,
         allow_guest_list_recipients=False,
     )
-    postage = None
     client_reference = None
-    if template.template_type == LETTER_TYPE:
-        # Validate address and set postage to europe|rest-of-world if international letter,
-        # otherwise persist_notification with use template postage
-        postage = validate_address(service, personalisation)
-        if not postage:
-            postage = template.postage
-        from app.utils import get_reference_from_personalisation
-        client_reference = get_reference_from_personalisation(personalisation)
 
     validate_created_by(service, post_data['created_by'])
 
@@ -115,23 +77,16 @@ def send_one_off_notification(service_id, post_data):
         created_by_id=post_data['created_by'],
         reply_to_text=reply_to,
         reference=create_one_off_reference(template.template_type),
-        postage=postage,
         client_reference=client_reference
     )
 
     queue_name = QueueNames.PRIORITY if template.process_type == PRIORITY else None
 
-    if template.template_type == LETTER_TYPE and service.research_mode:
-        _update_notification_status(
-            notification,
-            NOTIFICATION_DELIVERED,
-        )
-    else:
-        send_notification_to_queue(
-            notification=notification,
-            research_mode=service.research_mode,
-            queue=queue_name,
-        )
+    send_notification_to_queue(
+        notification=notification,
+        research_mode=service.research_mode,
+        queue=queue_name,
+    )
 
     return {'id': str(notification.id)}
 
@@ -151,72 +106,3 @@ def get_reply_to_text(notification_type, sender_id, service, template):
     else:
         reply_to = template.get_reply_to_text()
     return reply_to
-
-
-def send_pdf_letter_notification(service_id, post_data):
-    service = dao_fetch_service_by_id(service_id)
-
-    check_service_has_permission(LETTER_TYPE, [
-        p.permission for p in service.permissions
-    ])
-    check_service_over_daily_message_limit(KEY_TYPE_NORMAL, service)
-    validate_created_by(service, post_data['created_by'])
-    validate_and_format_recipient(
-        send_to=post_data['recipient_address'],
-        key_type=KEY_TYPE_NORMAL,
-        service=service,
-        notification_type=LETTER_TYPE,
-        allow_guest_list_recipients=False,
-    )
-
-    # notification already exists e.g. if the user clicked send in different tabs
-    if get_notification_by_id(post_data['file_id']):
-        return {'id': str(post_data['file_id'])}
-
-    template = get_precompiled_letter_template(service.id)
-    file_location = 'service-{}/{}.pdf'.format(service.id, post_data['file_id'])
-
-    try:
-        letter = utils_s3download(current_app.config['TRANSIENT_UPLOADED_LETTERS'], file_location)
-    except S3ObjectNotFound as e:
-        current_app.logger.warning('Letter {}.pdf not in transient {} bucket'.format(
-            post_data['file_id'], current_app.config['TRANSIENT_UPLOADED_LETTERS'])
-        )
-
-        raise e
-
-    # Getting the page count won't raise an error since admin has already checked the PDF is valid
-    page_count = get_page_count(letter.read())
-    billable_units = get_billable_units_for_letter_page_count(page_count)
-
-    personalisation = {
-        'address_line_1': post_data['filename']
-    }
-
-    notification = persist_notification(
-        notification_id=post_data['file_id'],
-        template_id=template.id,
-        template_version=template.version,
-        recipient=urllib.parse.unquote(post_data['recipient_address']),
-        service=service,
-        personalisation=personalisation,
-        notification_type=LETTER_TYPE,
-        api_key_id=None,
-        key_type=KEY_TYPE_NORMAL,
-        reference=create_one_off_reference(LETTER_TYPE),
-        client_reference=post_data['filename'],
-        created_by_id=post_data['created_by'],
-        billable_units=billable_units,
-        postage=post_data['postage'] or template.postage,
-    )
-
-    upload_filename = generate_letter_pdf_filename(
-        reference=notification.reference,
-        created_at=notification.created_at,
-        ignore_folder=False,
-        postage=notification.postage
-    )
-
-    move_uploaded_pdf_to_letters_bucket(file_location, upload_filename)
-
-    return {'id': str(notification.id)}

app/service/send_pdf_letter_schema.py

@@ -1,14 +0,0 @@
-send_pdf_letter_request = {
-    "$schema": "http://json-schema.org/draft-07/schema#",
-    "description": "POST send uploaded pdf letter",
-    "type": "object",
-    "title": "Send an uploaded pdf letter",
-    "properties": {
-        "postage": {"type": "string", "format": "postage"},
-        "filename": {"type": "string"},
-        "created_by": {"type": "string"},
-        "file_id": {"type": "string"},
-        "recipient_address": {"type": "string"}
-    },
-    "required": ["postage", "filename", "created_by", "file_id", "recipient_address"]
-}

app/service/service_data_retention_schema.py

@@ -5,7 +5,7 @@ add_service_data_retention_request = {
     "type": "object",
     "properties": {
         "days_of_retention": {"type": "integer"},
-        "notification_type": {"enum": ["sms", "letter", "email"]},
+        "notification_type": {"enum": ["sms", "email"]},
     },
     "required": ["days_of_retention", "notification_type"]
 }

app/service/service_senders_schema.py

@@ -13,19 +13,6 @@ add_service_email_reply_to_request = {
 }
 
 
-add_service_letter_contact_block_request = {
-    "$schema": "http://json-schema.org/draft-07/schema#",
-    "description": "POST service letter contact block",
-    "type": "object",
-    "title": "Add new letter contact block for service",
-    "properties": {
-        "contact_block": {"type": "string"},
-        "is_default": {"type": "boolean"}
-    },
-    "required": ["contact_block", "is_default"]
-}
-
-
 add_service_sms_sender_request = {
     "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST add service SMS sender",