Merge branch 'master' of https://github.com/alphagov/notifications-api into inboundsms_checkip

app/dao/services_dao.py

@@ -19,6 +19,7 @@ from app.models import (
     ApiKey,
     Template,
     TemplateHistory,
+    TemplateRedacted,
     Job,
     NotificationHistory,
     Notification,
@@ -110,6 +111,7 @@ def dao_archive_service(service_id):
     # to ensure that db.session still contains the models when it comes to creating history objects
     service = Service.query.options(
         joinedload('templates'),
+        joinedload('templates.template_redacted'),
         joinedload('api_keys'),
     ).filter(Service.id == service_id).one()
 
@@ -201,13 +203,16 @@ def dao_remove_user_from_service(service, user):
 def delete_service_and_all_associated_db_objects(service):
 
     def _delete_commit(query):
-        query.delete()
+        query.delete(synchronize_session=False)
         db.session.commit()
 
     job_stats = JobStatistics.query.join(Job).filter(Job.service_id == service.id)
     list(map(db.session.delete, job_stats))
     db.session.commit()
 
+    subq = db.session.query(Template.id).filter_by(service=service).subquery()
+    _delete_commit(TemplateRedacted.query.filter(TemplateRedacted.template_id.in_(subq)))
+
     _delete_commit(NotificationStatistics.query.filter_by(service=service))
     _delete_commit(TemplateStatistics.query.filter_by(service=service))
     _delete_commit(ProviderStatistics.query.filter_by(service=service))

app/dao/templates_dao.py

@@ -1,10 +1,11 @@
+from datetime import datetime
 import uuid
 
 from sqlalchemy import desc
 from sqlalchemy.sql.expression import bindparam
 
 from app import db
-from app.models import (Template, TemplateHistory)
+from app.models import (Template, TemplateHistory, TemplateRedacted)
 from app.dao.dao_utils import (
     transactional,
     version_class
@@ -16,6 +17,13 @@ from app.dao.dao_utils import (
 def dao_create_template(template):
     template.id = uuid.uuid4()  # must be set now so version history model can use same id
     template.archived = False
+
+    template.template_redacted = TemplateRedacted(
+        template=template,
+        redact_personalisation=False,
+        updated_by=template.created_by
+    )
+
     db.session.add(template)
 
 
@@ -25,6 +33,14 @@ def dao_update_template(template):
     db.session.add(template)
 
 
+@transactional
+def dao_redact_template(template, user_id):
+    template.template_redacted.redact_personalisation = True
+    template.template_redacted.updated_at = datetime.utcnow()
+    template.template_redacted.updated_by_id = user_id
+    db.session.add(template.template_redacted)
+
+
 def dao_get_template_by_id_and_service_id(template_id, service_id, version=None):
     if version is not None:
         return TemplateHistory.query.filter_by(

app/models.py

@@ -434,12 +434,16 @@ class Template(db.Model):
     subject = db.Column(db.Text, index=False, unique=False, nullable=True)
     created_by_id = db.Column(UUID(as_uuid=True), db.ForeignKey('users.id'), index=True, nullable=False)
     created_by = db.relationship('User')
-    version = db.Column(db.Integer, default=1, nullable=False)
-    process_type = db.Column(db.String(255),
-                             db.ForeignKey('template_process_type.name'),
-                             index=True,
-                             nullable=False,
-                             default=NORMAL)
+    version = db.Column(db.Integer, default=0, nullable=False)
+    process_type = db.Column(
+        db.String(255),
+        db.ForeignKey('template_process_type.name'),
+        index=True,
+        nullable=False,
+        default=NORMAL
+    )
+
+    redact_personalisation = association_proxy('template_redacted', 'redact_personalisation')
 
     def get_link(self):
         # TODO: use "/v2/" route once available
@@ -465,6 +469,19 @@ class Template(db.Model):
         return serialized
 
 
+class TemplateRedacted(db.Model):
+    __tablename__ = 'template_redacted'
+
+    template_id = db.Column(UUID(as_uuid=True), db.ForeignKey('templates.id'), primary_key=True, nullable=False)
+    redact_personalisation = db.Column(db.Boolean, nullable=False, default=False)
+    updated_at = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow)
+    updated_by_id = db.Column(UUID(as_uuid=True), db.ForeignKey('users.id'), nullable=False)
+    updated_by = db.relationship('User')
+
+    # uselist=False as this is a one-to-one relationship
+    template = db.relationship('Template', uselist=False, backref=db.backref('template_redacted', uselist=False))
+
+
 class TemplateHistory(db.Model):
     __tablename__ = 'templates_history'
 

app/schemas.py

@@ -297,6 +297,11 @@ class BaseTemplateSchema(BaseSchema):
         strict = True
 
 
+class TemplateRedactedSchema(BaseSchema):
+    class Meta:
+        model = models.TemplateRedacted
+
+
 class TemplateSchema(BaseTemplateSchema):
 
     created_by = field_for(models.Template, 'created_by', required=True)
@@ -440,7 +445,7 @@ class NotificationWithTemplateSchema(BaseSchema):
 
     template = fields.Nested(
         TemplateSchema,
-        only=['id', 'version', 'name', 'template_type', 'content', 'subject'],
+        only=['id', 'version', 'name', 'template_type', 'content', 'subject', 'redact_personalisation'],
         dump_only=True
     )
     job = fields.Nested(JobSchema, only=["id", "original_file_name"], dump_only=True)

app/template/rest.py

@@ -8,6 +8,7 @@ from flask import (
 from app.dao.templates_dao import (
     dao_update_template,
     dao_create_template,
+    dao_redact_template,
     dao_get_template_by_id_and_service_id,
     dao_get_all_templates_for_service,
     dao_get_template_versions
@@ -55,9 +56,15 @@ def create_template(service_id):
 def update_template(service_id, template_id):
     fetched_template = dao_get_template_by_id_and_service_id(template_id=template_id, service_id=service_id)
 
+    data = request.get_json()
+
+    # if redacting, don't update anything else
+    if data.get('redact_personalisation') is True:
+        return redact_template(fetched_template, data)
+
     current_data = dict(template_schema.dump(fetched_template).data.items())
     updated_template = dict(template_schema.dump(fetched_template).data.items())
-    updated_template.update(request.get_json())
+    updated_template.update(data)
     # Check if there is a change to make.
     if _template_has_not_changed(current_data, updated_template):
         return jsonify(data=updated_template), 200
@@ -132,3 +139,16 @@ def _template_has_not_changed(current_data, updated_template):
         current_data[key] == updated_template[key]
         for key in ('name', 'content', 'subject', 'archived', 'process_type')
     )
+
+
+def redact_template(template, data):
+    # we also don't need to check what was passed in redact_personalisation - its presence in the dict is enough.
+    if 'created_by' not in data:
+        message = 'Field is required'
+        errors = {'created_by': [message]}
+        raise InvalidRequest(errors, status_code=400)
+
+    # if it's already redacted, then just return 200 straight away.
+    if not template.redact_personalisation:
+        dao_redact_template(template, data['created_by'])
+    return 'null', 200

migrations/versions/0102_template_redacted.py

@@ -0,0 +1,31 @@
+"""empty message
+
+Revision ID: db6d9d9f06bc
+Revises: 0101_een_logo
+Create Date: 2017-06-27 15:37:28.878359
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'db6d9d9f06bc'
+down_revision = '0101_een_logo'
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+def upgrade():
+    op.create_table('template_redacted',
+        sa.Column('template_id', postgresql.UUID(as_uuid=True), nullable=False),
+        sa.Column('redact_personalisation', sa.Boolean(), nullable=False),
+        sa.Column('updated_at', sa.DateTime(), nullable=False),
+        sa.Column('updated_by_id', postgresql.UUID(as_uuid=True), nullable=False),
+        sa.ForeignKeyConstraint(['template_id'], ['templates.id'], ),
+        sa.ForeignKeyConstraint(['updated_by_id'], ['users.id'], ),
+        sa.PrimaryKeyConstraint('template_id')
+    )
+    op.create_index(op.f('ix_template_redacted_updated_by_id'), 'template_redacted', ['updated_by_id'], unique=False)
+
+
+def downgrade():
+    op.drop_table('template_redacted')

migrations/versions/0103_add_historical_redact.py

@@ -0,0 +1,43 @@
+"""empty message
+
+Revision ID: 0103_add_historical_redact
+Revises: db6d9d9f06bc
+Create Date: 2017-06-29 12:44:16.815039
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '0103_add_historical_redact'
+down_revision = 'db6d9d9f06bc'
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+from flask import current_app
+
+def upgrade():
+    op.execute(
+        """
+        INSERT INTO template_redacted
+        (
+            template_id,
+            redact_personalisation,
+            updated_at,
+            updated_by_id
+        )
+        SELECT
+            templates.id,
+            false,
+            now(),
+            '{notify_user}'
+        FROM
+            templates
+        LEFT JOIN template_redacted on template_redacted.template_id = templates.id
+        WHERE template_redacted.template_id IS NULL
+        """.format(notify_user=current_app.config['NOTIFY_USER_ID'])
+    )
+
+
+def downgrade():
+    # data migration, no downloads
+    pass

tests/app/dao/test_templates_dao.py

@@ -1,14 +1,21 @@
+from datetime import datetime
+
+from freezegun import freeze_time
 from sqlalchemy.orm.exc import NoResultFound
+import pytest
+
 from app.dao.templates_dao import (
     dao_create_template,
     dao_get_template_by_id_and_service_id,
     dao_get_all_templates_for_service,
     dao_update_template,
     dao_get_template_versions,
-    dao_get_templates_for_cache)
+    dao_get_templates_for_cache,
+    dao_redact_template)
+from app.models import Template, TemplateHistory, TemplateRedacted
+
 from tests.app.conftest import sample_template as create_sample_template
-from app.models import Template, TemplateHistory
-import pytest
+from tests.app.db import create_template
 
 
 @pytest.mark.parametrize('template_type, subject', [
@@ -35,6 +42,17 @@ def test_create_template(sample_service, sample_user, template_type, subject):
     assert dao_get_all_templates_for_service(sample_service.id)[0].process_type == 'normal'
 
 
+def test_create_template_creates_redact_entry(sample_service):
+    assert TemplateRedacted.query.count() == 0
+
+    template = create_template(sample_service)
+
+    redacted = TemplateRedacted.query.one()
+    assert redacted.template_id == template.id
+    assert redacted.redact_personalisation is False
+    assert redacted.updated_by_id == sample_service.created_by_id
+
+
 def test_update_template(sample_service, sample_user):
     data = {
         'name': 'Sample Template',
@@ -53,6 +71,20 @@ def test_update_template(sample_service, sample_user):
     assert dao_get_all_templates_for_service(sample_service.id)[0].name == 'new name'
 
 
+def test_redact_template(sample_template):
+    redacted = TemplateRedacted.query.one()
+    assert redacted.template_id == sample_template.id
+    assert redacted.redact_personalisation is False
+
+    time = datetime.now()
+    with freeze_time(time):
+        dao_redact_template(sample_template, sample_template.created_by_id)
+
+    assert redacted.redact_personalisation is True
+    assert redacted.updated_at == time
+    assert redacted.updated_by_id == sample_template.created_by_id
+
+
 def test_get_all_templates_for_service(notify_db, notify_db_session, service_factory):
     service_1 = service_factory.get('service 1', email_from='service.1')
     service_2 = service_factory.get('service 2', email_from='service.2')

tests/app/service/test_rest.py

@@ -11,6 +11,7 @@ from freezegun import freeze_time
 from app import encryption
 from app.dao.users_dao import save_model_user
 from app.dao.services_dao import dao_remove_user_from_service
+from app.dao.templates_dao import dao_redact_template
 from app.models import (
     User, Organisation, Rate, Service, ServicePermission, Notification,
     DVLA_ORG_LAND_REGISTRY,
@@ -19,7 +20,7 @@ from app.models import (
 )
 
 from tests import create_authorization_header
-from tests.app.db import create_template, create_service_inbound_api, create_user
+from tests.app.db import create_template, create_service_inbound_api, create_user, create_notification
 from tests.app.conftest import (
     sample_service as create_service,
     sample_user_service_permission as create_user_service_permission,
@@ -2263,3 +2264,37 @@ def test_send_one_off_notification(admin_request, sample_template, mocker):
 
     noti = Notification.query.one()
     assert response['id'] == str(noti.id)
+
+
+def test_get_notification_for_service_includes_template_redacted(admin_request, sample_notification):
+    resp = admin_request.get(
+        'service.get_notification_for_service',
+        service_id=sample_notification.service_id,
+        notification_id=sample_notification.id
+    )
+
+    assert resp['id'] == str(sample_notification.id)
+    assert resp['template']['redact_personalisation'] is False
+
+
+def test_get_all_notifications_for_service_includes_template_redacted(admin_request, sample_service):
+    normal_template = create_template(sample_service)
+
+    redacted_template = create_template(sample_service)
+    dao_redact_template(redacted_template, sample_service.created_by_id)
+
+    with freeze_time('2000-01-01'):
+        redacted_noti = create_notification(redacted_template)
+    with freeze_time('2000-01-02'):
+        normal_noti = create_notification(normal_template)
+
+    resp = admin_request.get(
+        'service.get_all_notifications_for_service',
+        service_id=sample_service.id
+    )
+
+    assert resp['notifications'][0]['id'] == str(normal_noti.id)
+    assert resp['notifications'][0]['template']['redact_personalisation'] is False
+
+    assert resp['notifications'][1]['id'] == str(redacted_noti.id)
+    assert resp['notifications'][1]['template']['redact_personalisation'] is True

tests/app/template/test_rest.py

@@ -1,11 +1,17 @@
+import uuid
 import json
 import random
 import string
+from datetime import datetime, timedelta
+
 import pytest
+from freezegun import freeze_time
+
 from app.models import Template
+from app.dao.templates_dao import dao_get_template_by_id, dao_redact_template
+
 from tests import create_authorization_header
 from tests.app.conftest import sample_template as create_sample_template
-from app.dao.templates_dao import dao_get_template_by_id
 
 
 @pytest.mark.parametrize('template_type, subject', [
@@ -539,3 +545,92 @@ def test_update_set_process_type_on_template(client, sample_template):
 
     template = dao_get_template_by_id(sample_template.id)
     assert template.process_type == 'priority'
+
+
+def test_update_redact_template(admin_request, sample_template):
+    assert sample_template.redact_personalisation is False
+
+    data = {
+        'redact_personalisation': True,
+        'created_by': str(sample_template.created_by_id)
+    }
+
+    dt = datetime.now()
+
+    with freeze_time(dt):
+        resp = admin_request.post(
+            'template.update_template',
+            service_id=sample_template.service_id,
+            template_id=sample_template.id,
+            _data=data
+        )
+
+    assert resp is None
+
+    assert sample_template.redact_personalisation is True
+    assert sample_template.template_redacted.updated_by_id == sample_template.created_by_id
+    assert sample_template.template_redacted.updated_at == dt
+
+    assert sample_template.version == 1
+
+
+def test_update_redact_template_ignores_other_properties(admin_request, sample_template):
+    data = {
+        'name': 'Foo',
+        'redact_personalisation': True,
+        'created_by': str(sample_template.created_by_id)
+    }
+
+    admin_request.post(
+        'template.update_template',
+        service_id=sample_template.service_id,
+        template_id=sample_template.id,
+        _data=data
+    )
+
+    assert sample_template.redact_personalisation is True
+    assert sample_template.name != 'Foo'
+
+
+def test_update_redact_template_does_nothing_if_already_redacted(admin_request, sample_template):
+    dt = datetime.now()
+    with freeze_time(dt):
+        dao_redact_template(sample_template, sample_template.created_by_id)
+
+    data = {
+        'redact_personalisation': True,
+        'created_by': str(sample_template.created_by_id)
+    }
+
+    with freeze_time(dt + timedelta(days=1)):
+        resp = admin_request.post(
+            'template.update_template',
+            service_id=sample_template.service_id,
+            template_id=sample_template.id,
+            _data=data
+        )
+
+    assert resp is None
+
+    assert sample_template.redact_personalisation is True
+    # make sure that it hasn't been updated
+    assert sample_template.template_redacted.updated_at == dt
+
+
+def test_update_redact_template_400s_if_no_created_by(admin_request, sample_template):
+    original_updated_time = sample_template.template_redacted.updated_at
+    resp = admin_request.post(
+        'template.update_template',
+        service_id=sample_template.service_id,
+        template_id=sample_template.id,
+        _data={'redact_personalisation': True},
+        _expected_status=400
+    )
+
+    assert resp == {
+        'result': 'error',
+        'message': {'created_by': ['Field is required']}
+    }
+
+    assert sample_template.redact_personalisation is False
+    assert sample_template.template_redacted.updated_at == original_updated_time