Merge pull request #1945 from alphagov/flexible-data-retention

Flexible data retention

app/dao/service_data_retention_dao.py

@@ -0,0 +1,44 @@
+from datetime import datetime
+
+from app import db
+from app.dao.dao_utils import transactional
+from app.models import ServiceDataRetention
+
+
+def fetch_service_data_retention_by_id(service_id, data_retention_id):
+    data_retention = ServiceDataRetention.query.filter_by(service_id=service_id, id=data_retention_id).first()
+    return data_retention
+
+
+def fetch_service_data_retention(service_id):
+    data_retention_list = ServiceDataRetention.query.filter_by(
+        service_id=service_id
+    ).order_by(
+        # in the order that models.notification_types are created (email, sms, letter)
+        ServiceDataRetention.notification_type
+    ).all()
+    return data_retention_list
+
+
+@transactional
+def insert_service_data_retention(service_id, notification_type, days_of_retention):
+    new_data_retention = ServiceDataRetention(service_id=service_id,
+                                              notification_type=notification_type,
+                                              days_of_retention=days_of_retention)
+
+    db.session.add(new_data_retention)
+    return new_data_retention
+
+
+@transactional
+def update_service_data_retention(service_data_retention_id, service_id, days_of_retention):
+    updated_count = ServiceDataRetention.query.filter(
+        ServiceDataRetention.id == service_data_retention_id,
+        ServiceDataRetention.service_id == service_id
+    ).update(
+        {
+            "days_of_retention": days_of_retention,
+            "updated_at": datetime.utcnow()
+        }
+    )
+    return updated_count

app/models.py

@@ -1859,3 +1859,14 @@ class ServiceDataRetention(db.Model):
     __table_args__ = (
         UniqueConstraint('service_id', 'notification_type', name='uix_service_data_retention'),
     )
+
+    def serialize(self):
+        return {
+            "id": str(self.id),
+            "service_id": str(self.service_id),
+            "service_name": self.service.name,
+            "notification_type": self.notification_type,
+            "days_of_retention": self.days_of_retention,
+            "created_at": self.created_at.strftime(DATETIME_FORMAT),
+            "updated_at": self.updated_at.strftime(DATETIME_FORMAT) if self.updated_at else None,
+        }

app/service/rest.py

@@ -24,6 +24,12 @@ from app.dao.fact_notification_status_dao import (
 )
 from app.dao.inbound_numbers_dao import dao_allocate_number_for_service
 from app.dao.organisation_dao import dao_get_organisation_by_service_id
+from app.dao.service_data_retention_dao import (
+    fetch_service_data_retention,
+    fetch_service_data_retention_by_id,
+    insert_service_data_retention,
+    update_service_data_retention,
+)
 from app.dao.service_sms_sender_dao import (
     archive_sms_sender,
     dao_add_sms_sender_for_service,
@@ -76,6 +82,10 @@ from app.errors import (
 from app.models import Service, EmailBranding
 from app.schema_validation import validate
 from app.service import statistics
+from app.service.service_data_retention_schema import (
+    add_service_data_retention_request,
+    update_service_data_retention_request
+)
 from app.service.service_senders_schema import (
     add_service_email_reply_to_request,
     add_service_letter_contact_block_request,
@@ -734,6 +744,54 @@ def is_service_name_unique():
     return jsonify(result=result), 200
 
 
+@service_blueprint.route('/<uuid:service_id>/data-retention', methods=['GET'])
+def get_data_retention_for_service(service_id):
+    data_retention_list = fetch_service_data_retention(service_id)
+    return jsonify([data_retention.serialize() for data_retention in data_retention_list]), 200
+
+
+@service_blueprint.route('/<uuid:service_id>/data-retention/<uuid:data_retention_id>', methods=['GET'])
+def get_data_retention_for_service_by_id(service_id, data_retention_id):
+    data_retention = fetch_service_data_retention_by_id(service_id, data_retention_id)
+    return jsonify(data_retention.serialize() if data_retention else {}), 200
+
+
+@service_blueprint.route('/<uuid:service_id>/data-retention', methods=['POST'])
+def create_service_data_retention(service_id):
+    form = validate(request.get_json(), add_service_data_retention_request)
+    try:
+        new_data_retention = insert_service_data_retention(
+            service_id=service_id,
+            notification_type=form.get("notification_type"),
+            days_of_retention=form.get("days_of_retention")
+        )
+    except IntegrityError:
+        raise InvalidRequest(
+            message="Service already has data retention for {} notification type".format(form.get("notification_type")),
+            status_code=400
+        )
+
+    return jsonify(result=new_data_retention.serialize()), 201
+
+
+@service_blueprint.route('/<uuid:service_id>/data-retention/<uuid:data_retention_id>', methods=['POST'])
+def modify_service_data_retention(service_id, data_retention_id):
+    form = validate(request.get_json(), update_service_data_retention_request)
+
+    update_count = update_service_data_retention(
+        service_data_retention_id=data_retention_id,
+        service_id=service_id,
+        days_of_retention=form.get("days_of_retention")
+    )
+    if update_count == 0:
+        raise InvalidRequest(
+            message="The service data retention for id: {} was not found for service: {}".format(data_retention_id,
+                                                                                                 service_id),
+            status_code=404)
+
+    return '', 204
+
+
 def check_request_args(request):
     service_id = request.args.get('service_id')
     name = request.args.get('name', None)

app/service/service_data_retention_schema.py

@@ -0,0 +1,23 @@
+add_service_data_retention_request = {
+    "$schema": "http://json-schema.org/draft-04/schema#",
+    "description": "POST service data retention schema",
+    "title": "Add service data retention for notification type api",
+    "type": "object",
+    "properties": {
+        "days_of_retention": {"type": "integer"},
+        "notification_type": {"enum": ["sms", "letter", "email"]},
+    },
+    "required": ["days_of_retention", "notification_type"]
+}
+
+
+update_service_data_retention_request = {
+    "$schema": "http://json-schema.org/draft-04/schema#",
+    "description": "POST service data retention schema",
+    "title": "Update service data retention for notification type api",
+    "type": "object",
+    "properties": {
+        "days_of_retention": {"type": "integer"},
+    },
+    "required": ["days_of_retention"]
+}

tests/app/dao/test_service_data_retention_dao.py

@@ -0,0 +1,133 @@
+import uuid
+from datetime import datetime
+
+import pytest
+from sqlalchemy.exc import IntegrityError
+
+from app.dao.service_data_retention_dao import (
+    fetch_service_data_retention,
+    insert_service_data_retention,
+    update_service_data_retention,
+    fetch_service_data_retention_by_id
+)
+from app.models import ServiceDataRetention
+from tests.app.db import create_service
+
+
+def test_fetch_service_data_retention(sample_service):
+    email_data_retention = insert_service_data_retention(sample_service.id, 'email', 3)
+    letter_data_retention = insert_service_data_retention(sample_service.id, 'letter', 30)
+    sms_data_retention = insert_service_data_retention(sample_service.id, 'sms', 5)
+
+    list_of_data_retention = fetch_service_data_retention(sample_service.id)
+
+    assert len(list_of_data_retention) == 3
+    assert list_of_data_retention[0] == email_data_retention
+    assert list_of_data_retention[1] == sms_data_retention
+    assert list_of_data_retention[2] == letter_data_retention
+
+
+def test_fetch_service_data_retention_only_returns_row_for_service(sample_service):
+    another_service = create_service(service_name="Another service")
+    email_data_retention = insert_service_data_retention(sample_service.id, 'email', 3)
+    letter_data_retention = insert_service_data_retention(sample_service.id, 'letter', 30)
+    insert_service_data_retention(another_service.id, 'sms', 5)
+
+    list_of_data_retention = fetch_service_data_retention(sample_service.id)
+    assert len(list_of_data_retention) == 2
+    assert list_of_data_retention[0] == email_data_retention
+    assert list_of_data_retention[1] == letter_data_retention
+
+
+def test_fetch_service_data_retention_returns_empty_list_when_no_rows_for_service(sample_service):
+    empty_list = fetch_service_data_retention(sample_service.id)
+    assert not empty_list
+
+
+def test_fetch_service_data_retention_by_id(sample_service):
+    email_data_retention = insert_service_data_retention(sample_service.id, 'email', 3)
+    insert_service_data_retention(sample_service.id, 'sms', 13)
+    result = fetch_service_data_retention_by_id(sample_service.id, email_data_retention.id)
+    assert result == email_data_retention
+
+
+def test_fetch_service_data_retention_by_id_returns_none_if_not_found(sample_service):
+    result = fetch_service_data_retention_by_id(sample_service.id, uuid.uuid4())
+    assert not result
+
+
+def test_fetch_service_data_retention_by_id_returns_none_if_id_not_for_service(sample_service):
+    another_service = create_service(service_name="Another service")
+    email_data_retention = insert_service_data_retention(sample_service.id, 'email', 3)
+    result = fetch_service_data_retention_by_id(another_service.id, email_data_retention.id)
+    assert not result
+
+
+def test_insert_service_data_retention(sample_service):
+    insert_service_data_retention(
+        service_id=sample_service.id,
+        notification_type='email',
+        days_of_retention=3
+    )
+
+    results = ServiceDataRetention.query.all()
+    assert len(results) == 1
+    assert results[0].service_id == sample_service.id
+    assert results[0].notification_type == 'email'
+    assert results[0].days_of_retention == 3
+    assert results[0].created_at.date() == datetime.utcnow().date()
+
+
+def test_insert_service_data_retention_throws_unique_constraint(sample_service):
+    insert_service_data_retention(service_id=sample_service.id,
+                                  notification_type='email',
+                                  days_of_retention=3
+                                  )
+    with pytest.raises(expected_exception=IntegrityError):
+        insert_service_data_retention(service_id=sample_service.id,
+                                      notification_type='email',
+                                      days_of_retention=5
+                                      )
+
+
+def test_update_service_data_retention(sample_service):
+    data_retention = insert_service_data_retention(service_id=sample_service.id,
+                                                   notification_type='sms',
+                                                   days_of_retention=3
+                                                   )
+    updated_count = update_service_data_retention(service_data_retention_id=data_retention.id,
+                                                  service_id=sample_service.id,
+                                                  days_of_retention=5
+                                                  )
+    assert updated_count == 1
+    results = ServiceDataRetention.query.all()
+    assert len(results) == 1
+    assert results[0].id == data_retention.id
+    assert results[0].service_id == sample_service.id
+    assert results[0].notification_type == 'sms'
+    assert results[0].days_of_retention == 5
+    assert results[0].created_at.date() == datetime.utcnow().date()
+    assert results[0].updated_at.date() == datetime.utcnow().date()
+
+
+def test_update_service_data_retention_does_not_update_if_row_does_not_exist(sample_service):
+    updated_count = update_service_data_retention(
+        service_data_retention_id=uuid.uuid4(),
+        service_id=sample_service.id,
+        days_of_retention=5
+    )
+    assert updated_count == 0
+    assert len(ServiceDataRetention.query.all()) == 0
+
+
+def test_update_service_data_retention_does_not_update_row_if_data_retention_is_for_different_service(
+        sample_service
+):
+    data_retention = insert_service_data_retention(service_id=sample_service.id,
+                                                   notification_type='email',
+                                                   days_of_retention=3
+                                                   )
+    updated_count = update_service_data_retention(service_data_retention_id=data_retention.id,
+                                                  service_id=uuid.uuid4(),
+                                                  days_of_retention=5)
+    assert updated_count == 0

tests/app/db.py

@@ -3,6 +3,7 @@ import uuid
 
 from app import db
 from app.dao.jobs_dao import dao_create_job
+from app.dao.service_data_retention_dao import insert_service_data_retention
 from app.dao.service_inbound_api_dao import save_service_inbound_api
 from app.dao.service_callback_api_dao import save_service_callback_api
 from app.dao.service_sms_sender_dao import update_existing_sms_sender_with_inbound_number, dao_update_service_sms_sender
@@ -35,7 +36,7 @@ from app.models import (
     InvitedOrganisationUser,
     FactBilling,
     FactNotificationStatus,
-    Complaint
+    Complaint,
 )
 from app.dao.users_dao import save_model_user
 from app.dao.notifications_dao import (
@@ -663,3 +664,16 @@ def ses_notification_callback():
            'dd426d95ee9390147a5624348ee.pem",' \
            '\n  "UnsubscribeURL" : "https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&S' \
            'subscriptionArn=arn:aws:sns:eu-west-1:302763885840:preview-emails:d6aad3ef-83d6-4cf3-a470-54e2e75916da"\n}'
+
+
+def create_service_data_retention(
+        service_id,
+        notification_type='sms',
+        days_of_retention=3
+):
+    data_retention = insert_service_data_retention(
+        service_id=service_id,
+        notification_type=notification_type,
+        days_of_retention=days_of_retention
+    )
+    return data_retention

tests/app/service/test_service_data_retention_rest.py

@@ -0,0 +1,152 @@
+import json
+import uuid
+
+from app.models import ServiceDataRetention
+from tests import create_authorization_header
+from tests.app.db import create_service_data_retention
+
+
+def test_get_service_data_retention(client, sample_service):
+    sms_data_retention = create_service_data_retention(service_id=sample_service.id)
+    email_data_retention = create_service_data_retention(service_id=sample_service.id, notification_type='email',
+                                                         days_of_retention=10)
+    letter_data_retention = create_service_data_retention(service_id=sample_service.id, notification_type='letter',
+                                                          days_of_retention=30)
+
+    response = client.get(
+        '/service/{}/data-retention'.format(str(sample_service.id)),
+        headers=[('Content-Type', 'application/json'), create_authorization_header()],
+    )
+
+    assert response.status_code == 200
+    json_response = json.loads(response.get_data(as_text=True))
+    assert len(json_response) == 3
+    assert json_response[0] == email_data_retention.serialize()
+    assert json_response[1] == sms_data_retention.serialize()
+    assert json_response[2] == letter_data_retention.serialize()
+
+
+def test_get_service_data_retention_returns_empty_list(client, sample_service):
+    response = client.get(
+        '/service/{}/data-retention'.format(str(sample_service.id)),
+        headers=[('Content-Type', 'application/json'), create_authorization_header()],
+    )
+    assert response.status_code == 200
+    assert len(json.loads(response.get_data(as_text=True))) == 0
+
+
+def test_get_service_data_retention_by_id(client, sample_service):
+    sms_data_retention = create_service_data_retention(service_id=sample_service.id)
+    create_service_data_retention(service_id=sample_service.id, notification_type='email',
+                                  days_of_retention=10)
+    create_service_data_retention(service_id=sample_service.id, notification_type='letter',
+                                  days_of_retention=30)
+    response = client.get(
+        '/service/{}/data-retention/{}'.format(str(sample_service.id), sms_data_retention.id),
+        headers=[('Content-Type', 'application/json'), create_authorization_header()],
+    )
+    assert response.status_code == 200
+    assert json.loads(response.get_data(as_text=True)) == sms_data_retention.serialize()
+
+
+def test_get_service_data_retention_by_id_returns_none_when_no_data_retention_exists(client, sample_service):
+    response = client.get(
+        '/service/{}/data-retention/{}'.format(str(sample_service.id), uuid.uuid4()),
+        headers=[('Content-Type', 'application/json'), create_authorization_header()],
+    )
+    assert response.status_code == 200
+    assert json.loads(response.get_data(as_text=True)) == {}
+
+
+def test_create_service_data_retention(client, sample_service):
+    data = {
+        "notification_type": 'sms',
+        "days_of_retention": 3
+    }
+    response = client.post(
+        '/service/{}/data-retention'.format(str(sample_service.id)),
+        headers=[('Content-Type', 'application/json'), create_authorization_header()],
+        data=json.dumps(data)
+    )
+
+    assert response.status_code == 201
+    json_resp = json.loads(response.get_data(as_text=True))['result']
+    results = ServiceDataRetention.query.all()
+    assert len(results) == 1
+    data_retention = results[0]
+    assert json_resp == data_retention.serialize()
+
+
+def test_create_service_data_retention_returns_400_when_notification_type_is_invalid(client):
+    data = {
+        "notification_type": 'unknown',
+        "days_of_retention": 3
+    }
+    response = client.post(
+        '/service/{}/data-retention'.format(str(uuid.uuid4())),
+        headers=[('Content-Type', 'application/json'), create_authorization_header()],
+        data=json.dumps(data)
+    )
+    json_resp = json.loads(response.get_data(as_text=True))
+    assert response.status_code == 400
+    assert json_resp['errors'][0]['error'] == 'ValidationError'
+    assert json_resp['errors'][0]['message'] == 'notification_type unknown is not one of [sms, letter, email]'
+
+
+def test_create_service_data_retention_returns_400_when_data_retention_for_notification_type_already_exists(
+        client, sample_service
+):
+    create_service_data_retention(service_id=sample_service.id)
+    data = {
+        "notification_type": "sms",
+        "days_of_retention": 3
+    }
+    response = client.post(
+        '/service/{}/data-retention'.format(str(uuid.uuid4())),
+        headers=[('Content-Type', 'application/json'), create_authorization_header()],
+        data=json.dumps(data)
+    )
+
+    assert response.status_code == 400
+    json_resp = json.loads(response.get_data(as_text=True))
+    assert json_resp['result'] == 'error'
+    assert json_resp['message'] == 'Service already has data retention for sms notification type'
+
+
+def test_modify_service_data_retention(client, sample_service):
+    data_retention = create_service_data_retention(service_id=sample_service.id)
+    data = {
+        "days_of_retention": 3
+    }
+    response = client.post(
+        '/service/{}/data-retention/{}'.format(sample_service.id, data_retention.id),
+        headers=[('Content-Type', 'application/json'), create_authorization_header()],
+        data=json.dumps(data)
+    )
+    assert response.status_code == 204
+    assert response.get_data(as_text=True) == ''
+
+
+def test_modify_service_data_retention_returns_400_when_data_retention_does_not_exist(client, sample_service):
+    data = {
+        "days_of_retention": 3
+    }
+    response = client.post(
+        '/service/{}/data-retention/{}'.format(sample_service.id, uuid.uuid4()),
+        headers=[('Content-Type', 'application/json'), create_authorization_header()],
+        data=json.dumps(data)
+    )
+
+    assert response.status_code == 404
+
+
+def test_modify_service_data_retention_returns_400_when_data_is_invalid(client):
+    data = {
+        "bad_key": 3
+    }
+    response = client.post(
+        '/service/{}/data-retention/{}'.format(uuid.uuid4(), uuid.uuid4()),
+        headers=[('Content-Type', 'application/json'), create_authorization_header()],
+        data=json.dumps(data)
+    )
+    assert response.status_code == 400