Merge pull request #2676 from alphagov/add-returned-letters-table

Add returned letters table

app/celery/tasks.py

@@ -43,6 +43,7 @@ from app.dao.notifications_dao import (
     dao_get_notification_history_by_reference,
 )
 from app.dao.provider_details_dao import get_provider_details_by_notification_type
+from app.dao.returned_letters_dao import insert_or_update_returned_letters
 from app.dao.service_email_reply_to_dao import dao_get_reply_to_by_id
 from app.dao.service_inbound_api_dao import get_service_inbound_api_for_service
 from app.dao.service_sms_sender_dao import dao_get_service_sms_senders_by_id
@@ -619,6 +620,8 @@ def process_returned_letters_list(notification_references):
         {"status": NOTIFICATION_RETURNED_LETTER}
     )
 
+    insert_or_update_returned_letters(notification_references)
+
     current_app.logger.info(
         "Updated {} letter notifications ({} history notifications, from {} references) to returned-letter".format(
             updated, updated_history, len(notification_references)

app/dao/returned_letters_dao.py

@@ -0,0 +1,56 @@
+from datetime import datetime
+
+from sqlalchemy import func, desc
+from sqlalchemy.dialects.postgresql import insert
+
+from app import db
+from app.dao.dao_utils import transactional
+from app.models import Notification, NotificationHistory, ReturnedLetter
+
+
+def _get_notification_ids_for_references(references):
+    notification_ids = db.session.query(Notification.id, Notification.service_id).filter(
+        Notification.reference.in_(references)
+    ).all()
+
+    notification_history_ids = db.session.query(NotificationHistory.id, NotificationHistory.service_id).filter(
+        NotificationHistory.reference.in_(references)
+    ).all()
+
+    return notification_ids + notification_history_ids
+
+
+@transactional
+def insert_or_update_returned_letters(references):
+    data = _get_notification_ids_for_references(references)
+    for row in data:
+        table = ReturnedLetter.__table__
+
+        stmt = insert(table).values(
+            reported_at=datetime.utcnow().date(),
+            service_id=row.service_id,
+            notification_id=row.id,
+            created_at=datetime.utcnow()
+        )
+
+        stmt = stmt.on_conflict_do_update(
+            index_elements=[table.c.notification_id],
+            set_={
+                'reported_at': datetime.utcnow().date(),
+                'updated_at': datetime.utcnow()
+            }
+        )
+        db.session.connection().execute(stmt)
+
+
+def get_returned_letter_summary(service_id):
+    return db.session.query(
+        func.count(ReturnedLetter.notification_id).label('returned_letter_count'),
+        ReturnedLetter.reported_at
+    ).filter(
+        ReturnedLetter.service_id == service_id,
+    ).group_by(
+        ReturnedLetter.reported_at
+    ).order_by(
+        desc(ReturnedLetter.reported_at)
+    ).all()

app/models.py

@@ -2019,8 +2019,7 @@ class Complaint(db.Model):
     __tablename__ = 'complaints'
 
     id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
-    notification_id = db.Column(UUID(as_uuid=True), db.ForeignKey('notification_history.id'),
-                                index=True, nullable=False)
+    notification_id = db.Column(UUID(as_uuid=True), index=True, nullable=False)
     service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'), unique=False, index=True, nullable=False)
     service = db.relationship(Service, backref=db.backref('complaints'))
     ses_feedback_id = db.Column(db.Text, nullable=True)
@@ -2072,3 +2071,15 @@ class ServiceDataRetention(db.Model):
             "created_at": self.created_at.strftime(DATETIME_FORMAT),
             "updated_at": self.updated_at.strftime(DATETIME_FORMAT) if self.updated_at else None,
         }
+
+
+class ReturnedLetter(db.Model):
+    __tablename__ = 'returned_letters'
+
+    id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+    reported_at = db.Column(db.Date, nullable=False)
+    service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'), unique=False, index=True, nullable=False)
+    service = db.relationship(Service, backref=db.backref('returned_letters'))
+    notification_id = db.Column(UUID(as_uuid=True), unique=True, nullable=False)
+    created_at = db.Column(db.DateTime, nullable=False)
+    updated_at = db.Column(db.DateTime, nullable=True, onupdate=datetime.datetime.utcnow)

app/service/rest.py

@@ -12,6 +12,7 @@ from notifications_utils.timezones import convert_utc_to_bst
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm.exc import NoResultFound
 
+from app import DATE_FORMAT
 from app.config import QueueNames
 from app.dao import fact_notification_status_dao, notifications_dao
 from app.dao.dao_utils import dao_rollback
@@ -29,6 +30,7 @@ from app.dao.fact_notification_status_dao import (
 )
 from app.dao.inbound_numbers_dao import dao_allocate_number_for_service
 from app.dao.organisation_dao import dao_get_organisation_by_service_id
+from app.dao.returned_letters_dao import get_returned_letter_summary
 from app.dao.service_data_retention_dao import (
     fetch_service_data_retention,
     fetch_service_data_retention_by_id,
@@ -939,3 +941,14 @@ def check_if_reply_to_address_already_in_use(service_id, email_address):
         raise InvalidRequest(
             "Your service already uses ‘{}’ as an email reply-to address.".format(email_address), status_code=400
         )
+
+
+@service_blueprint.route('/<uuid:service_id>/returned-letter-summary', methods=['GET'])
+def returned_letter_summary(service_id):
+    results = get_returned_letter_summary(service_id)
+
+    json_results = [{'returned_letter_count': x.returned_letter_count,
+                     'reported_at': x.reported_at.strftime(DATE_FORMAT)
+                     } for x in results]
+
+    return jsonify(json_results)

migrations/versions/0310_returned_letters_table_.py

@@ -0,0 +1,32 @@
+"""
+
+Revision ID: 0310_returned_letters_table
+Revises: 0309_add_uq_key_row_number
+Create Date: 2019-12-09 12:13:49.432993
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+revision = '0310_returned_letters_table'
+down_revision = '0309_add_uq_key_row_number'
+
+
+def upgrade():
+    op.create_table('returned_letters',
+        sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False),
+        sa.Column('reported_at', sa.Date(), nullable=False),
+        sa.Column('service_id', postgresql.UUID(as_uuid=True), nullable=False),
+        sa.Column('notification_id', postgresql.UUID(as_uuid=True), nullable=False),
+        sa.Column('created_at', sa.DateTime(), nullable=False),
+        sa.Column('updated_at', sa.DateTime(), nullable=True),
+        sa.ForeignKeyConstraint(['service_id'], ['services.id'], ),
+        sa.PrimaryKeyConstraint('id'),
+        sa.UniqueConstraint('notification_id')
+    )
+    op.create_index(op.f('ix_returned_letters_service_id'), 'returned_letters', ['service_id'], unique=False)
+
+
+def downgrade():
+    op.drop_table('returned_letters')

tests/app/celery/test_tasks.py

@@ -44,6 +44,7 @@ from app.models import (
     JOB_STATUS_IN_PROGRESS,
     LETTER_TYPE,
     SMS_TYPE,
+    ReturnedLetter
 )
 
 from tests.app import load_example_csv
@@ -1626,3 +1627,15 @@ def test_process_returned_letters_list_updates_history_if_notification_is_alread
 
     assert [n.status for n in notifications] == ['returned-letter', 'returned-letter']
     assert all(n.updated_at for n in notifications)
+
+
+def test_process_returned_letters_populates_returned_letters_table(
+        sample_letter_template
+):
+    create_notification_history(sample_letter_template, reference='ref1')
+    create_notification_history(sample_letter_template, reference='ref2')
+
+    process_returned_letters_list(['ref1', 'ref2', 'unknown-ref'])
+
+    returned_letters = ReturnedLetter.query.all()
+    assert len(returned_letters) == 2

tests/app/dao/test_returned_letters_dao.py

@@ -0,0 +1,118 @@
+from datetime import datetime, timedelta, date
+
+from freezegun import freeze_time
+
+from app.dao.returned_letters_dao import insert_or_update_returned_letters, get_returned_letter_summary
+from app.models import ReturnedLetter
+from tests.app.db import create_notification, create_notification_history, create_returned_letter
+
+
+def test_insert_or_update_returned_letters_inserts(sample_letter_template):
+    notification = create_notification(template=sample_letter_template,
+                                       reference='ref1')
+    history = create_notification_history(template=sample_letter_template,
+                                          reference='ref2')
+
+    assert ReturnedLetter.query.count() == 0
+
+    insert_or_update_returned_letters(['ref1', 'ref2'])
+
+    returned_letters = ReturnedLetter.query.all()
+
+    assert len(returned_letters) == 2
+    returned_letters_ = [x.notification_id for x in returned_letters]
+    assert notification.id in returned_letters_
+    assert history.id in returned_letters_
+
+
+def test_insert_or_update_returned_letters_updates(sample_letter_template):
+    notification = create_notification(template=sample_letter_template,
+                                       reference='ref1')
+    history = create_notification_history(template=sample_letter_template,
+                                          reference='ref2')
+
+    assert ReturnedLetter.query.count() == 0
+    with freeze_time('2019-12-09 13:30'):
+        insert_or_update_returned_letters(['ref1', 'ref2'])
+        returned_letters = ReturnedLetter.query.all()
+        assert len(returned_letters) == 2
+        for x in returned_letters:
+            assert x.reported_at == date(2019, 12, 9)
+            assert x.created_at == datetime(2019, 12, 9, 13, 30)
+            assert not x.updated_at
+            assert x.notification_id in [notification.id, history.id]
+
+    with freeze_time('2019-12-10 14:20'):
+        insert_or_update_returned_letters(['ref1', 'ref2'])
+        returned_letters = ReturnedLetter.query.all()
+        assert len(returned_letters) == 2
+        for x in returned_letters:
+            assert x.reported_at == date(2019, 12, 10)
+            assert x.created_at == datetime(2019, 12, 9, 13, 30)
+            assert x.updated_at == datetime(2019, 12, 10, 14, 20)
+            assert x.notification_id in [notification.id, history.id]
+
+
+def test_insert_or_update_returned_letters_when_no_notification(notify_db_session):
+    insert_or_update_returned_letters(['ref1'])
+    assert ReturnedLetter.query.count() == 0
+
+
+def test_insert_or_update_returned_letters_for_history_only(sample_letter_template):
+    history_1 = create_notification_history(template=sample_letter_template,
+                                            reference='ref1')
+    history_2 = create_notification_history(template=sample_letter_template,
+                                            reference='ref2')
+
+    assert ReturnedLetter.query.count() == 0
+    insert_or_update_returned_letters(['ref1', 'ref2'])
+    returned_letters = ReturnedLetter.query.all()
+    assert len(returned_letters) == 2
+    for x in returned_letters:
+        assert x.notification_id in [history_1.id, history_2.id]
+
+
+def test_insert_or_update_returned_letters_with_duplicates_in_reference_list(sample_letter_template):
+    notification_1 = create_notification(template=sample_letter_template,
+                                         reference='ref1')
+    notification_2 = create_notification(template=sample_letter_template,
+                                         reference='ref2')
+
+    assert ReturnedLetter.query.count() == 0
+    insert_or_update_returned_letters(['ref1', 'ref2', 'ref1', 'ref2'])
+    returned_letters = ReturnedLetter.query.all()
+    assert len(returned_letters) == 2
+    for x in returned_letters:
+        assert x.notification_id in [notification_1.id, notification_2.id]
+
+
+def test_get_returned_letter_summary(sample_service):
+    now = datetime.utcnow()
+    create_returned_letter(sample_service, reported_at=now)
+    create_returned_letter(sample_service, reported_at=now)
+
+    results = get_returned_letter_summary(sample_service.id)
+
+    assert len(results) == 1
+
+    assert results[0].returned_letter_count == 2
+    assert results[0].reported_at == now.date()
+
+
+def test_get_returned_letter_summary_orders_by_reported_at(sample_service):
+    now = datetime.utcnow()
+    last_month = datetime.utcnow() - timedelta(days=30)
+    create_returned_letter(sample_service, reported_at=now)
+    create_returned_letter(sample_service, reported_at=now)
+    create_returned_letter(sample_service, reported_at=now)
+    create_returned_letter(sample_service, reported_at=last_month)
+    create_returned_letter(sample_service, reported_at=last_month)
+    create_returned_letter()  # returned letter for a different service
+
+    results = get_returned_letter_summary(sample_service.id)
+
+    assert len(results) == 2
+    assert results[0].reported_at == now.date()
+    assert results[0].returned_letter_count == 3
+    assert results[1].reported_at == last_month.date()
+    assert results[1].returned_letter_count == 2

tests/app/db.py

@@ -59,7 +59,9 @@ from app.models import (
     TemplateFolder,
     LetterBranding,
     Domain,
-    NotificationHistory
+    NotificationHistory,
+    NOTIFICATION_RETURNED_LETTER,
+    ReturnedLetter
 )
 
 
@@ -940,3 +942,20 @@ def set_up_usage_data(start_date):
                       notifications_sent=15, billable_unit=4, rate=.55, postage='second')
 
     return org, org_3, service, service_3, service_4, service_sms_only
+
+
+def create_returned_letter(service=None, reported_at=None):
+    if not service:
+        service = create_service(service_name='a - with sms and letter')
+    template = create_template(service=service, template_type=LETTER_TYPE)
+    notification = create_notification(template=template, status=NOTIFICATION_RETURNED_LETTER)
+    returned_letter = ReturnedLetter(
+        service_id=service.id,
+        reported_at=reported_at or datetime.utcnow(),
+        notification_id=notification.id,
+        created_at=datetime.utcnow(),
+    )
+
+    db.session.add(returned_letter)
+    db.session.commit()
+    return returned_letter

tests/app/service/test_rest.py

@@ -47,7 +47,7 @@ from tests.app.db import (
     create_domain,
     create_email_branding,
     create_annual_billing,
-)
+    create_returned_letter)
 from tests.app.db import create_user
 
 
@@ -3372,3 +3372,16 @@ def test_get_monthly_notification_data_by_service(mocker, admin_request):
 
     dao_mock.assert_called_once_with(start_date, end_date)
     assert response == []
+
+
+@freeze_time('2019-12-11 13:30')
+def test_get_returned_letter_summary(admin_request, sample_service):
+    create_returned_letter(sample_service, reported_at=datetime.utcnow() - timedelta(days=3))
+    create_returned_letter(sample_service, reported_at=datetime.utcnow())
+    create_returned_letter(sample_service, reported_at=datetime.utcnow())
+
+    response = admin_request.get('service.returned_letter_summary', service_id=sample_service.id)
+
+    assert len(response) == 2
+    assert response[0] == {'returned_letter_count': 2, 'reported_at': '2019-12-11'}
+    assert response[1] == {'returned_letter_count': 1, 'reported_at': '2019-12-08'}