Merge pull request #2171 from alphagov/flexible-data-retention

Flexible data retention

app/__init__.py

@@ -344,6 +344,14 @@ def valid_phone_number(phone_number):
         return False
 
 
+def format_notification_type(notification_type):
+    return {
+        'email': 'Email',
+        'sms': 'SMS',
+        'letter': 'Letter'
+    }[notification_type]
+
+
 def format_notification_status(status, template_type):
     return {
         'email': {
@@ -646,6 +654,7 @@ def add_template_filters(application):
         format_datetime_relative,
         format_delta,
         format_notification_status,
+        format_notification_type,
         format_notification_status_as_time,
         format_notification_status_as_field_status,
         format_notification_status_as_url,

app/main/forms.py

@@ -1035,3 +1035,27 @@ class BrandingOptionsEmail(StripWhitespaceForm):
             DataRequired()
         ],
     )
+
+
+class ServiceDataRetentionForm(StripWhitespaceForm):
+
+    notification_type = RadioField(
+        'What notification type?',
+        choices=[
+            ('email', 'Email'),
+            ('sms', 'SMS'),
+            ('letter', 'Letter'),
+        ],
+        validators=[DataRequired()],
+    )
+    days_of_retention = IntegerField(label="Days of retention",
+                                     validators=[validators.NumberRange(min=3, max=90,
+                                                                        message="Must be between 3 and 90")],
+                                     )
+
+
+class ServiceDataRetentionEditForm(StripWhitespaceForm):
+    days_of_retention = IntegerField(label="Days of retention",
+                                     validators=[validators.NumberRange(min=3, max=90,
+                                                                        message="Must be between 3 and 90")],
+                                     )

app/main/views/service_settings.py

@@ -36,6 +36,8 @@ from app.main.forms import (
     RequestToGoLiveForm,
     ServiceBasicViewForm,
     ServiceContactLinkForm,
+    ServiceDataRetentionEditForm,
+    ServiceDataRetentionForm,
     ServiceEditInboundNumberForm,
     ServiceInboundNumberForm,
     ServiceLetterContactBlockForm,
@@ -86,6 +88,7 @@ def service_settings(service_id):
     )
 
     free_sms_fragment_limit = billing_api_client.get_free_sms_fragment_limit_for_year(service_id)
+    data_retention = service_api_client.get_service_data_retention(service_id)
 
     return render_template(
         'views/service-settings.html',
@@ -104,6 +107,7 @@ def service_settings(service_id):
         free_sms_fragment_limit=free_sms_fragment_limit,
         prefix_sms=current_service.prefix_sms,
         organisation=organisation,
+        data_retention=data_retention,
     )
 
 
@@ -1006,6 +1010,48 @@ def branding_request(service_id):
     )
 
 
+@main.route("/services/<service_id>/data-retention", methods=['GET'])
+@login_required
+@user_is_platform_admin
+def data_retention(service_id):
+    results = service_api_client.get_service_data_retention(service_id)
+    return render_template('views/service-settings/data-retention.html',
+                           data_retention_settings=results)
+
+
+@main.route("/services/<service_id>/data-retention/add", methods=['GET', 'POST'])
+@login_required
+@user_is_platform_admin
+def add_data_retention(service_id):
+    form = ServiceDataRetentionForm()
+    if form.validate_on_submit():
+        service_api_client.create_service_data_retention(service_id,
+                                                         form.notification_type.data,
+                                                         form.days_of_retention.data)
+        return redirect(url_for('.data_retention', service_id=service_id))
+    return render_template(
+        'views/service-settings/data-retention/add.html',
+        form=form
+    )
+
+
+@main.route("/services/<service_id>/data-retention/<data_retention_id>/edit", methods=['GET', 'POST'])
+@login_required
+@user_is_platform_admin
+def edit_data_retention(service_id, data_retention_id):
+    data_retention_item = service_api_client.get_service_data_retention_by_id(service_id, data_retention_id)
+    form = ServiceDataRetentionEditForm(days_of_retention=data_retention_item['days_of_retention'])
+    if form.validate_on_submit():
+        service_api_client.update_service_data_retention(service_id, data_retention_id, form.days_of_retention.data)
+        return redirect(url_for('.data_retention', service_id=service_id))
+    return render_template(
+        'views/service-settings/data-retention/edit.html',
+        form=form,
+        data_retention_id=data_retention_id,
+        notification_type=data_retention_item['notification_type']
+    )
+
+
 def get_branding_as_value_and_label(email_branding):
     return [
         (branding['id'], branding['name'])

app/navigation.py

@@ -103,6 +103,7 @@ class HeaderNavigation(Navigation):
         'accept_invite',
         'accept_org_invite',
         'action_blocked',
+        'add_data_retention',
         'add_service',
         'add_service_template',
         'add_template_by_type',
@@ -135,12 +136,14 @@ class HeaderNavigation(Navigation):
         'cookies',
         'copy_template',
         'create_api_key',
+        'data_retention',
         'delete_service_template',
         'delivery_and_failure',
         'delivery_status_callback',
         'design_content',
         'download_agreement',
         'download_notifications_csv',
+        'edit_data_retention',
         'edit_organisation_name',
         'edit_provider',
         'edit_service_template',
@@ -372,6 +375,7 @@ class MainNavigation(Navigation):
     exclude = {
         'accept_invite',
         'accept_org_invite',
+        'add_data_retention',
         'add_organisation',
         'add_service',
         'agreement',
@@ -391,11 +395,13 @@ class MainNavigation(Navigation):
         'conversation_updates',
         'cookies',
         'create_email_branding',
+        'data_retention',
         'delivery_and_failure',
         'design_content',
         'documentation',
         'download_agreement',
         'download_notifications_csv',
+        'edit_data_retention',
         'edit_organisation_name',
         'edit_provider',
         'edit_user_org_permissions',
@@ -529,6 +535,7 @@ class CaseworkNavigation(Navigation):
         'accept_invite',
         'accept_org_invite',
         'action_blocked',
+        'add_data_retention',
         'add_organisation',
         'add_service',
         'add_service_template',
@@ -563,6 +570,7 @@ class CaseworkNavigation(Navigation):
         'copy_template',
         'create_api_key',
         'create_email_branding',
+        'data_retention',
         'delete_service_template',
         'delivery_and_failure',
         'delivery_status_callback',
@@ -570,6 +578,7 @@ class CaseworkNavigation(Navigation):
         'documentation',
         'download_agreement',
         'download_notifications_csv',
+        'edit_data_retention',
         'edit_organisation_name',
         'edit_provider',
         'edit_service_template',
@@ -755,6 +764,7 @@ class OrgNavigation(Navigation):
         'accept_invite',
         'accept_org_invite',
         'action_blocked',
+        'add_data_retention',
         'add_organisation',
         'add_service',
         'add_service_template',
@@ -789,6 +799,7 @@ class OrgNavigation(Navigation):
         'copy_template',
         'create_api_key',
         'create_email_branding',
+        'data_retention',
         'delete_service_template',
         'delivery_and_failure',
         'delivery_status_callback',
@@ -796,6 +807,7 @@ class OrgNavigation(Navigation):
         'documentation',
         'download_agreement',
         'download_notifications_csv',
+        'edit_data_retention',
         'edit_provider',
         'edit_service_template',
         'edit_user_permissions',

app/notify_client/service_api_client.py

@@ -480,3 +480,23 @@ class ServiceAPIClient(NotifyAdminAPIClient):
             "updated_by_id": user_id
         }
         return self.post("/service/{}/delivery-receipt-api".format(service_id), data)
+
+    def create_service_data_retention(self, service_id, notification_type, days_of_retention):
+        data = {
+            "notification_type": notification_type,
+            "days_of_retention": days_of_retention
+        }
+
+        return self.post("/service/{}/data-retention".format(service_id), data)
+
+    def update_service_data_retention(self, service_id, data_retention_id, days_of_retention):
+        data = {
+            "days_of_retention": days_of_retention
+        }
+        return self.post("/service/{}/data-retention/{}".format(service_id, data_retention_id), data)
+
+    def get_service_data_retention(self, service_id):
+        return self.get("/service/{}/data-retention".format(service_id))
+
+    def get_service_data_retention_by_id(self, service_id, data_retention_id):
+        return self.get("service/{}/data-retention/{}".format(service_id, data_retention_id))

app/templates/views/service-settings.html

@@ -320,6 +320,13 @@
           {{ text_field(letter_branding) }}
           {{ edit_field('Change', url_for('.set_letter_branding', service_id=current_service.id)) }}
         {% endcall %}
+        {% call row() %}
+          {{ text_field('Data retention')}}
+          {% call field() %}
+              {{ data_retention | join(', ', attribute='notification_type') }}
+          {% endcall %}
+          {{ edit_field('Change', url_for('.data_retention', service_id=current_service.id)) }}
+        {% endcall %}
       {% endcall %}
 
       <ul>

app/templates/views/service-settings/data-retention.html

@@ -0,0 +1,50 @@
+{% extends "withnav_template.html" %}
+{% from "components/api-key.html" import api_key %}
+{% from "components/page-footer.html" import page_footer %}
+{% from "components/table.html" import mapping_table, row_group, row, text_field, edit_field, field, boolean_field, list_table with context %}
+
+{% block service_page_title %}
+  Email reply to addresses
+{% endblock %}
+
+{% block maincolumn_content %}
+
+  <div class="grid-row bottom-gutter">
+    <div class="column-two-thirds">
+      <h1 class="heading-large">
+        Data retention
+      </h1>
+    </div>
+   <div class="column-one-third">
+	    <a href="{{ url_for('.add_data_retention', service_id=current_service.id) }}" class="button align-with-heading">Add data retention</a>
+  </div>
+  </div>
+  <div class="grid-row bottom-gutter">
+      <div class="column-full">
+          By default data is kept for 7 days
+      </div>
+  </div>
+  <div class="user-list">
+    {% if not data_retention_settings %}
+      <div class="user-list-item">
+        <span class="hint">You haven’t added any data retention</span>
+      </div>
+    {% endif %}
+       {% call mapping_table(
+        caption='Data retention',
+        field_headings=['Label', 'Value', 'Action'],
+        field_headings_visible=False,
+        caption_visible=False
+      ) %}
+        {% for item in data_retention_settings %}
+            {% call row() %}
+              {{ text_field(item.notification_type | format_notification_type)}}
+              {{ text_field(item.days_of_retention) }}
+              {{ edit_field('Change', url_for('.edit_data_retention', service_id=current_service.id, data_retention_id=item.id)) }}
+            {% endcall %}
+
+        {% endfor %}
+      {% endcall %}
+  </div>
+
+{% endblock %}

app/templates/views/service-settings/data-retention/add.html

@@ -0,0 +1,25 @@
+{% extends "withnav_template.html" %}
+{% from "components/textbox.html" import textbox %}
+{% from "components/page-footer.html" import page_footer %}
+
+{% from "components/radios.html" import radios %}
+
+{% block service_page_title %}
+  Data retention
+{% endblock %}
+
+{% block maincolumn_content %}
+
+
+  <h1 class="heading-large">Set data retention</h1>
+    <form method="post">
+      {{ radios(form.notification_type) }}
+      {{  textbox(form.days_of_retention) }}
+      {{ page_footer(
+      'Add',
+      back_link=url_for('.add_data_retention', service_id=current_service.id),
+      back_link_text='Back'
+    ) }}
+    </form>
+
+{% endblock %}

app/templates/views/service-settings/data-retention/edit.html

@@ -0,0 +1,25 @@
+{% extends "withnav_template.html" %}
+{% from "components/textbox.html" import textbox %}
+{% from "components/page-footer.html" import page_footer %}
+
+{% from "components/radios.html" import radios %}
+
+{% block service_page_title %}
+  Data retention
+{% endblock %}
+
+{% block maincolumn_content %}
+
+
+  <h1 class="heading-large">Set data retention</h1>
+    <form method="post">
+      {{ notification_type | capitalize}}
+      {{  textbox(form.days_of_retention) }}
+      {{ page_footer(
+      'Save',
+      back_link=url_for('.edit_data_retention', service_id=current_service.id, data_retention_id=data_retention_id),
+      back_link_text='Back'
+    ) }}
+    </form>
+
+{% endblock %}

tests/app/main/views/service_settings/test_service_setting_permissions.py

@@ -16,6 +16,7 @@ def get_service_settings_page(
     no_reply_to_email_addresses,
     no_letter_contact_blocks,
     single_sms_sender,
+    mock_get_service_data_retention,
 ):
     client_request.login(platform_admin_user)
     return functools.partial(client_request.get, 'main.service_settings', service_id=service_one['id'])
@@ -103,6 +104,7 @@ def test_normal_user_doesnt_see_any_toggle_buttons(
     mock_get_letter_email_branding,
     mock_get_inbound_number_for_service,
     mock_get_free_sms_fragment_limit,
+    mock_get_service_data_retention
 ):
     page = client_request.get('main.service_settings', service_id=service_one['id'])
     toggles = page.find('a', {'class': 'button'})

tests/app/main/views/test_service_settings.py

@@ -40,6 +40,7 @@ def mock_get_service_settings_page_common(
     mock_get_letter_email_branding,
     mock_get_inbound_number_for_service,
     mock_get_free_sms_fragment_limit,
+    mock_get_service_data_retention,
 ):
     return
 
@@ -94,6 +95,7 @@ def mock_get_service_settings_page_common(
         'Free text message allowance 250,000 Change',
         'Email branding GOV.UK Change',
         'Letter branding HM Government Change',
+        'Data retention email Change'
 
     ]),
 ])
@@ -2531,6 +2533,7 @@ def test_service_settings_when_inbound_number_is_not_set(
     mocker,
     mock_get_letter_email_branding,
     mock_get_free_sms_fragment_limit,
+    mock_get_service_data_retention,
 ):
     mocker.patch('app.inbound_number_client.get_inbound_sms_number_for_service',
                  return_value={'data': {}})
@@ -2794,3 +2797,98 @@ def test_submit_email_branding_request(
         'Thanks for your branding request. We’ll get back to you '
         'within one working day.'
     )
+
+
+def test_show_service_data_retention(
+        logged_in_platform_admin_client,
+        service_one,
+        mock_get_service_data_retention,
+
+):
+    response = logged_in_platform_admin_client.get(url_for('main.data_retention', service_id=service_one['id']))
+    page = BeautifulSoup(response.data.decode('utf-8'), 'html.parser')
+    rows = page.select('tbody tr')
+    assert len(rows) == 1
+    assert normalize_spaces(rows[0].text) == 'Email 5 Change'
+
+
+def test_view_add_service_data_retention(
+        logged_in_platform_admin_client,
+        service_one,
+
+):
+    response = logged_in_platform_admin_client.get(url_for('main.add_data_retention', service_id=service_one['id']))
+    page = BeautifulSoup(response.data.decode('utf-8'), 'html.parser')
+    assert normalize_spaces(page.select_one('input')['value']) == "email"
+    assert page.find('input', attrs={'name': 'days_of_retention'})
+
+
+def test_add_service_data_retention(
+        logged_in_platform_admin_client,
+        service_one,
+        mock_create_service_data_retention
+):
+    response = logged_in_platform_admin_client.post(url_for('main.add_data_retention', service_id=service_one['id']),
+                                                    data={'notification_type': "email",
+                                                          'days_of_retention': 5
+                                                          }
+                                                    )
+    assert response.status_code == 302
+    settings_url = url_for(
+        'main.data_retention', service_id=service_one['id'], _external=True)
+    assert settings_url == response.location
+    assert mock_create_service_data_retention.called
+
+
+def test_update_service_data_retention(
+        logged_in_platform_admin_client,
+        service_one,
+        fake_uuid,
+        mock_get_service_data_retention_by_id,
+        mock_update_service_data_retention,
+):
+    response = logged_in_platform_admin_client.post(url_for('main.edit_data_retention',
+                                                            service_id=service_one['id'],
+                                                            data_retention_id=fake_uuid),
+                                                    data={'days_of_retention': 5}
+                                                    )
+    assert response.status_code == 302
+    settings_url = url_for(
+        'main.data_retention', service_id=service_one['id'], _external=True)
+    assert settings_url == response.location
+    assert mock_update_service_data_retention.called
+
+
+def test_update_service_data_retention_return_validation_error_for_negative_days_of_retention(
+        logged_in_platform_admin_client,
+        service_one,
+        fake_uuid,
+        mock_get_service_data_retention_by_id,
+        mock_update_service_data_retention,
+):
+    response = logged_in_platform_admin_client.post(url_for('main.edit_data_retention',
+                                                            service_id=service_one['id'],
+                                                            data_retention_id=fake_uuid),
+                                                    data={'days_of_retention': -5}
+                                                    )
+    assert response.status_code == 200
+    page = BeautifulSoup(response.data.decode('utf-8'), 'html.parser')
+    error_message = page.find('span', class_='error-message').text.strip()
+    assert error_message == 'Must be between 3 and 90'
+    assert mock_get_service_data_retention_by_id.called
+    assert not mock_update_service_data_retention.called
+
+
+def test_update_service_data_retention_populates_form(
+        logged_in_platform_admin_client,
+        service_one,
+        fake_uuid,
+        mock_get_service_data_retention_by_id,
+):
+    response = logged_in_platform_admin_client.get(url_for('main.edit_data_retention',
+                                                           service_id=service_one['id'],
+                                                           data_retention_id=fake_uuid)
+                                                   )
+    assert response.status_code == 200
+    page = BeautifulSoup(response.data.decode('utf-8'), 'html.parser')
+    assert page.find('input', attrs={'name': 'days_of_retention'})['value'] == '5'

tests/conftest.py

@@ -2684,6 +2684,44 @@ def normalize_spaces(input):
     return normalize_spaces(' '.join(item.text for item in input))
 
 
+@pytest.fixture(scope='function')
+def mock_get_service_data_retention(mocker):
+    data = {"id": str(fake_uuid),
+            "service_id": str(fake_uuid),
+            "service_name": "service name",
+            "notification_type": "email",
+            "days_of_retention": 5,
+            "created_at": datetime.now(),
+            "updated_at": None,
+            }
+    return mocker.patch('app.service_api_client.get_service_data_retention',
+                        return_value=[data])
+
+
+@pytest.fixture(scope='function')
+def mock_get_service_data_retention_by_id(mocker):
+    data = {"id": str(fake_uuid),
+            "service_id": str(fake_uuid),
+            "service_name": "service name",
+            "notification_type": "email",
+            "days_of_retention": 5,
+            "created_at": datetime.now(),
+            "updated_at": None,
+            }
+    return mocker.patch('app.service_api_client.get_service_data_retention_by_id',
+                        return_value=data)
+
+
+@pytest.fixture(scope='function')
+def mock_create_service_data_retention(mocker):
+    return mocker.patch('app.service_api_client.create_service_data_retention')
+
+
+@pytest.fixture(scope='function')
+def mock_update_service_data_retention(mocker):
+    return mocker.patch('app.service_api_client.update_service_data_retention')
+
+
 @pytest.fixture(scope='function')
 def mock_get_free_sms_fragment_limit(mocker):
     sample_limit = 250000