Degrade inbound number not associated with service message from error to warning

Endpoint to return a ServiceContactList for a given id.

app/celery/letters_pdf_tasks.py

@@ -120,7 +120,6 @@ def get_letters_pdf(template, contact_block, filename, values):
 
 @notify_celery.task(name='collate-letter-pdfs-to-be-sent')
 @cronitor("collate-letter-pdfs-to-be-sent")
-@statsd(namespace="tasks")
 def collate_letter_pdfs_to_be_sent():
     """
     Finds all letters which are still waiting to be sent to DVLA for printing

app/celery/scheduled_tasks.py

@@ -232,13 +232,6 @@ def check_precompiled_letter_state():
 @notify_celery.task(name='check-templated-letter-state')
 @statsd(namespace="tasks")
 def check_templated_letter_state():
-    # renaming task for clarity. Next deploy will remove this task.
-    check_letters_created_yesterday_are_sending()
-
-
-@notify_celery.task(name='check-letters-created-yesterday-are-sending')
-@statsd(namespace="tasks")
-def check_letters_created_yesterday_are_sending():
     letters = dao_old_letters_with_created_status()
 
     if len(letters) > 0:

app/dao/notifications_dao.py

@@ -731,7 +731,6 @@ def notifications_not_yet_sent(should_be_sending_after_seconds, notification_typ
     return notifications
 
 
-@statsd(namespace="dao")
 def dao_get_letters_to_be_printed(print_run_deadline):
     """
     Return all letters created before the print run deadline that have not yet been sent

app/dao/service_contact_list_dao.py

@@ -0,0 +1,25 @@
+from app import db
+from app.models import ServiceContactList
+
+
+def dao_get_contact_list_by_id(service_id, contact_list_id):
+    contact_list = ServiceContactList.query.filter_by(
+        service_id=service_id,
+        id=contact_list_id
+    ).one()
+
+    return contact_list
+
+
+def dao_get_contact_lists(service_id):
+    contact_lists = ServiceContactList.query.filter_by(
+        service_id=service_id
+    ).order_by(
+        ServiceContactList.created_at.desc()
+    )
+    return contact_lists.all()
+
+
+def save_service_contact_list(service_contact_list):
+    db.session.add(service_contact_list)
+    db.session.commit()

app/models.py

@@ -1214,6 +1214,7 @@ class Job(db.Model):
         db.String(255), db.ForeignKey('job_status.name'), index=True, nullable=False, default='pending'
     )
     archived = db.Column(db.Boolean, nullable=False, default=False)
+    contact_list_id = db.Column(UUID(as_uuid=True), db.ForeignKey('service_contact_list.id'), nullable=True)
 
 
 VERIFY_CODE_TYPES = [EMAIL_TYPE, SMS_TYPE]
@@ -2119,3 +2120,31 @@ class ReturnedLetter(db.Model):
     notification_id = db.Column(UUID(as_uuid=True), unique=True, nullable=False)
     created_at = db.Column(db.DateTime, nullable=False)
     updated_at = db.Column(db.DateTime, nullable=True, onupdate=datetime.datetime.utcnow)
+
+
+class ServiceContactList(db.Model):
+    __tablename__ = 'service_contact_list'
+
+    id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+    original_file_name = db.Column(db.String, nullable=False)
+    row_count = db.Column(db.Integer, nullable=False)
+    template_type = db.Column(template_types, nullable=False)
+    service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'), unique=False, index=True, nullable=False)
+    service = db.relationship(Service, backref=db.backref('contact_list'))
+    created_by = db.relationship('User')
+    created_by_id = db.Column(UUID(as_uuid=True), db.ForeignKey('users.id'), index=True, nullable=True)
+    created_at = db.Column(db.DateTime, nullable=False)
+    updated_at = db.Column(db.DateTime, nullable=True, onupdate=datetime.datetime.utcnow)
+
+    def serialize(self):
+        created_at_in_bst = convert_utc_to_bst(self.created_at)
+        contact_list = {
+            "id": str(self.id),
+            "original_file_name": self.original_file_name,
+            "row_count": self.row_count,
+            "template_type": self.template_type,
+            "service_id": str(self.service_id),
+            "created_by": self.created_by.name,
+            "created_at": created_at_in_bst.strftime("%Y-%m-%d %H:%M:%S"),
+        }
+        return contact_list

app/notifications/receive_notifications.py

@@ -149,7 +149,7 @@ def fetch_potential_service(inbound_number, provider_name):
     service = dao_fetch_service_by_inbound_number(inbound_number)
 
     if not service:
-        current_app.logger.error('Inbound number "{}" from {} not associated with a service'.format(
+        current_app.logger.warning('Inbound number "{}" from {} not associated with a service'.format(
             inbound_number, provider_name
         ))
         statsd_client.incr('inbound.{}.failed'.format(provider_name))

app/service/rest.py

@@ -36,6 +36,8 @@ from app.dao.returned_letters_dao import (
     fetch_returned_letter_summary,
     fetch_returned_letters,
 )
+from app.dao.service_contact_list_dao import dao_get_contact_lists, save_service_contact_list, \
+    dao_get_contact_list_by_id
 from app.dao.service_data_retention_dao import (
     fetch_service_data_retention,
     fetch_service_data_retention_by_id,
@@ -95,12 +97,14 @@ from app.errors import (
 from app.letters.utils import letter_print_day
 from app.models import (
     KEY_TYPE_NORMAL, LETTER_TYPE, NOTIFICATION_CANCELLED, Permission, Service,
-    EmailBranding, LetterBranding
+    EmailBranding, LetterBranding,
+    ServiceContactList
 )
 from app.notifications.process_notifications import persist_notification, send_notification_to_queue
 from app.schema_validation import validate
 from app.service import statistics
 from app.service.send_pdf_letter_schema import send_pdf_letter_request
+from app.service.service_contact_list_schema import create_service_contact_list_schema
 from app.service.service_data_retention_schema import (
     add_service_data_retention_request,
     update_service_data_retention_request
@@ -1011,3 +1015,33 @@ def get_returned_letters(service_id):
          } for x in results]
 
     return jsonify(sorted(json_results, key=lambda i: i['created_at'], reverse=True))
+
+
+@service_blueprint.route('/<uuid:service_id>/contact-list', methods=['GET'])
+def get_contact_list(service_id):
+    contact_lists = dao_get_contact_lists(service_id)
+
+    return jsonify([x.serialize() for x in contact_lists])
+
+
+@service_blueprint.route('/<uuid:service_id>/contact-list/<uuid:contact_list_id>', methods=['GET'])
+def get_contact_list_by_id(service_id, contact_list_id):
+    contact_list = dao_get_contact_list_by_id(
+        service_id=service_id,
+        contact_list_id=contact_list_id
+    )
+
+    return jsonify(contact_list.serialize())
+
+
+@service_blueprint.route('/<uuid:service_id>/contact-list', methods=['POST'])
+def create_contact_list(service_id):
+    service_contact_list = validate(request.get_json(), create_service_contact_list_schema)
+    service_contact_list['created_by_id'] = service_contact_list.pop('created_by')
+    service_contact_list['created_at'] = datetime.utcnow()
+    service_contact_list['service_id'] = str(service_id)
+    list_to_save = ServiceContactList(**service_contact_list)
+
+    save_service_contact_list(list_to_save)
+
+    return jsonify(list_to_save.serialize()), 201

app/service/service_contact_list_schema.py

@@ -0,0 +1,16 @@
+from app.schema_validation.definitions import uuid
+
+create_service_contact_list_schema = {
+    "$schema": "http://json-schema.org/draft-04/schema#",
+    "description": "POST create service contact list schema",
+    "type": "object",
+    "title": "Create service contact list",
+    "properties": {
+        "id": uuid,
+        "original_file_name": {"type": "string"},
+        "row_count": {"type": "integer"},
+        "template_type": {"enum": ['email', 'sms']},
+        "created_by": uuid
+    },
+    "required": ["id", "original_file_name", "row_count", "template_type", "created_by"]
+}

migrations/versions/0318_service_contact_list.py

@@ -0,0 +1,42 @@
+"""
+
+Revision ID: 0318_service_contact_list
+Revises: 0317_uploads_for_all
+Create Date: 2020-03-12 15:44:30.784031
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+revision = '0318_service_contact_list'
+down_revision = '0317_uploads_for_all'
+
+
+def upgrade():
+    op.create_table(
+        'service_contact_list',
+        sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False),
+        sa.Column('original_file_name', sa.String(), nullable=False),
+        sa.Column('row_count', sa.Integer(), nullable=False),
+        sa.Column('template_type', postgresql.ENUM(name='template_type', create_type=False), nullable=False),
+        sa.Column('service_id', postgresql.UUID(as_uuid=True), nullable=False),
+        sa.Column('created_by_id', postgresql.UUID(as_uuid=True), nullable=True),
+        sa.Column('created_at', sa.DateTime(), nullable=False),
+        sa.Column('updated_at', sa.DateTime(), nullable=True),
+        sa.ForeignKeyConstraint(['created_by_id'], ['users.id'], ),
+        sa.ForeignKeyConstraint(['service_id'], ['services.id'], ),
+        sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_service_contact_list_created_by_id'), 'service_contact_list', ['created_by_id'], unique=False)
+    op.create_index(op.f('ix_service_contact_list_service_id'), 'service_contact_list', ['service_id'], unique=False)
+    op.add_column('jobs', sa.Column('contact_list_id', postgresql.UUID(as_uuid=True), nullable=True))
+    op.create_foreign_key('jobs_contact_list_id_fkey', 'jobs', 'service_contact_list', ['contact_list_id'], ['id'])
+
+
+def downgrade():
+    op.drop_constraint('jobs_contact_list_id_fkey', 'jobs', type_='foreignkey')
+    op.drop_column('jobs', 'contact_list_id')
+    op.drop_index(op.f('ix_service_contact_list_service_id'), table_name='service_contact_list')
+    op.drop_index(op.f('ix_service_contact_list_created_by_id'), table_name='service_contact_list')
+    op.drop_table('service_contact_list')

tests/app/celery/test_scheduled_tasks.py

@@ -15,10 +15,10 @@ from app.celery.scheduled_tasks import (
     send_scheduled_notifications,
     replay_created_notifications,
     check_precompiled_letter_state,
+    check_templated_letter_state,
     check_for_missing_rows_in_completed_jobs,
     check_for_services_with_high_failure_rates_or_sending_to_tv_numbers,
     switch_current_sms_provider_on_slow_delivery,
-    check_letters_created_yesterday_are_sending
 )
 from app.config import QueueNames, TaskNames, Config
 from app.dao.jobs_dao import dao_get_job_by_id
@@ -385,7 +385,7 @@ def test_check_precompiled_letter_state(mocker, sample_letter_template):
 
 
 @freeze_time("2019-05-30 14:00:00")
-def test_check_letters_created_yesterday_are_sending(mocker, sample_letter_template):
+def test_check_templated_letter_state_during_bst(mocker, sample_letter_template):
     mock_logger = mocker.patch('app.celery.tasks.current_app.logger.exception')
     mock_create_ticket = mocker.patch('app.celery.nightly_tasks.zendesk_client.create_ticket')
 
@@ -396,7 +396,7 @@ def test_check_letters_created_yesterday_are_sending(mocker, sample_letter_templ
     create_notification(template=sample_letter_template, status='delivered', created_at=datetime(2019, 5, 28, 10, 0))
     create_notification(template=sample_letter_template, created_at=datetime(2019, 5, 30, 10, 0))
 
-    check_letters_created_yesterday_are_sending()
+    check_templated_letter_state()
 
     message = "2 letters were created before 17.30 yesterday and still have 'created' status. " \
               "Notifications: ['{}', '{}']".format(noti_1.id, noti_2.id)
@@ -410,7 +410,7 @@ def test_check_letters_created_yesterday_are_sending(mocker, sample_letter_templ
 
 
 @freeze_time("2019-01-30 14:00:00")
-def test_check_letters_created_yesterday_are_sending_during_utc(mocker, sample_letter_template):
+def test_check_templated_letter_state_during_utc(mocker, sample_letter_template):
     mock_logger = mocker.patch('app.celery.tasks.current_app.logger.exception')
     mock_create_ticket = mocker.patch('app.celery.scheduled_tasks.zendesk_client.create_ticket')
 
@@ -421,7 +421,7 @@ def test_check_letters_created_yesterday_are_sending_during_utc(mocker, sample_l
     create_notification(template=sample_letter_template, status='delivered', created_at=datetime(2019, 1, 29, 10, 0))
     create_notification(template=sample_letter_template, created_at=datetime(2019, 1, 30, 10, 0))
 
-    check_letters_created_yesterday_are_sending()
+    check_templated_letter_state()
 
     message = "2 letters were created before 17.30 yesterday and still have 'created' status. " \
               "Notifications: ['{}', '{}']".format(noti_1.id, noti_2.id)

tests/app/dao/test_service_contact_list_dao.py

@@ -0,0 +1,10 @@
+from app.dao.service_contact_list_dao import dao_get_contact_lists
+from tests.app.db import create_service_contact_list
+
+
+def test_dao_get_contact_lists(notify_db_session):
+    contact_list = create_service_contact_list()
+    fetched_list = dao_get_contact_lists(contact_list.service_id)
+
+    assert len(fetched_list) == 1
+    assert fetched_list[0] == contact_list

tests/app/db.py

@@ -60,7 +60,8 @@ from app.models import (
     LetterBranding,
     Domain,
     NotificationHistory,
-    ReturnedLetter
+    ReturnedLetter,
+    ServiceContactList
 )
 
 
@@ -965,3 +966,26 @@ def create_returned_letter(service=None, reported_at=None, notification_id=None)
     db.session.add(returned_letter)
     db.session.commit()
     return returned_letter
+
+
+def create_service_contact_list(
+    service=None,
+    original_file_name='EmergencyContactList.xls',
+    row_count=100,
+    template_type='email',
+    created_by_id=None
+):
+    if not service:
+        service = create_service(service_name='service for contact list', user=create_user())
+
+    contact_list = ServiceContactList(
+        service_id=service.id,
+        original_file_name=original_file_name,
+        row_count=row_count,
+        template_type=template_type,
+        created_by_id=created_by_id or service.users[0].id,
+        created_at=datetime.utcnow(),
+    )
+    db.session.add(contact_list)
+    db.session.commit()
+    return contact_list

tests/app/service/test_service_contact_list_rest.py

@@ -0,0 +1,115 @@
+import uuid
+
+from app.models import ServiceContactList
+from tests.app.db import create_service_contact_list, create_service
+
+
+def test_create_service_contact_list(sample_service, admin_request):
+    data = {
+        "id": str(uuid.uuid4()),
+        "row_count": 100,
+        "original_file_name": "staff_emergency_list.xls",
+        "template_type": 'email',
+        "created_by": str(sample_service.users[0].id)
+    }
+
+    response = admin_request.post(
+        'service.create_contact_list',
+        _data=data,
+        service_id=sample_service.id,
+        _expected_status=201
+    )
+
+    assert response['id'] == data['id']
+    assert response['original_file_name'] == 'staff_emergency_list.xls'
+    assert response['row_count'] == 100
+    assert response['template_type'] == 'email'
+    assert response['service_id'] == str(sample_service.id)
+    assert response['created_at']
+
+    db_results = ServiceContactList.query.all()
+    assert len(db_results) == 1
+    assert str(db_results[0].id) == data['id']
+
+
+def test_create_service_contact_list_cannot_save_type_letter(sample_service, admin_request):
+    data = {
+        "id": str(uuid.uuid4()),
+        "row_count": 100,
+        "original_file_name": "staff_emergency_list.xls",
+        "template_type": 'letter',
+        "created_by": str(sample_service.users[0].id)
+    }
+
+    response = admin_request.post(
+        'service.create_contact_list',
+        _data=data,
+        service_id=sample_service.id,
+        _expected_status=400
+    )
+    assert response['errors'][0]['message'] == "template_type letter is not one of [email, sms]"
+
+
+def test_get_contact_list(admin_request, notify_db_session):
+    contact_list = create_service_contact_list()
+
+    response = admin_request.get(
+        'service.get_contact_list',
+        service_id=contact_list.service_id
+    )
+
+    assert len(response) == 1
+    assert response[0] == contact_list.serialize()
+
+
+def test_get_contact_list_returns_for_service(admin_request, notify_db_session):
+    service_1 = create_service(service_name='Service under test')
+    service_2 = create_service(service_name='Service should return results')
+
+    expected_list_1 = create_service_contact_list(service=service_1)
+    expected_list_2 = create_service_contact_list(service=service_1)
+    # not included in results
+    create_service_contact_list(service=service_2)
+
+    response = admin_request.get(
+        'service.get_contact_list',
+        service_id=service_1.id
+    )
+
+    assert len(response) == 2
+    assert response[0] == expected_list_2.serialize()
+    assert response[1] == expected_list_1.serialize()
+
+
+def test_dao_get_contact_list_by_id(admin_request, sample_service):
+    service_1 = create_service(service_name='Service under test')
+
+    expected_list_1 = create_service_contact_list(service=service_1)
+    create_service_contact_list(service=service_1)
+
+    response = admin_request.get(
+        'service.get_contact_list_by_id',
+        service_id=service_1.id,
+        contact_list_id=expected_list_1.id
+    )
+
+    assert response == expected_list_1.serialize()
+
+
+def test_dao_get_contact_list_by_id_does_not_return_if_contact_list_id_for_another_service(
+    admin_request, sample_service
+):
+    service_1 = create_service(service_name='Service requesting list')
+    service_2 = create_service(service_name='Service that owns the list')
+
+    create_service_contact_list(service=service_1)
+    list_2 = create_service_contact_list(service=service_2)
+
+    response = admin_request.get(
+        'service.get_contact_list_by_id',
+        service_id=service_1.id,
+        contact_list_id=list_2.id,
+        _expected_status=404
+    )
+
+    assert response['message'] == "No result found"