diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 24c95567a..7429a1e10 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -19,6 +19,7 @@ from app.models import ( ApiKey, Template, TemplateHistory, + TemplateRedacted, Job, NotificationHistory, Notification, @@ -110,6 +111,7 @@ def dao_archive_service(service_id): # to ensure that db.session still contains the models when it comes to creating history objects service = Service.query.options( joinedload('templates'), + joinedload('templates.template_redacted'), joinedload('api_keys'), ).filter(Service.id == service_id).one() @@ -201,13 +203,16 @@ def dao_remove_user_from_service(service, user): def delete_service_and_all_associated_db_objects(service): def _delete_commit(query): - query.delete() + query.delete(synchronize_session=False) db.session.commit() job_stats = JobStatistics.query.join(Job).filter(Job.service_id == service.id) list(map(db.session.delete, job_stats)) db.session.commit() + subq = db.session.query(Template.id).filter_by(service=service).subquery() + _delete_commit(TemplateRedacted.query.filter(TemplateRedacted.template_id.in_(subq))) + _delete_commit(NotificationStatistics.query.filter_by(service=service)) _delete_commit(TemplateStatistics.query.filter_by(service=service)) _delete_commit(ProviderStatistics.query.filter_by(service=service)) diff --git a/app/dao/templates_dao.py b/app/dao/templates_dao.py index 493c7c072..6cae74c0e 100644 --- a/app/dao/templates_dao.py +++ b/app/dao/templates_dao.py @@ -1,10 +1,11 @@ +from datetime import datetime import uuid from sqlalchemy import desc from sqlalchemy.sql.expression import bindparam from app import db -from app.models import (Template, TemplateHistory) +from app.models import (Template, TemplateHistory, TemplateRedacted) from app.dao.dao_utils import ( transactional, version_class @@ -16,6 +17,13 @@ from app.dao.dao_utils import ( def dao_create_template(template): template.id = uuid.uuid4() # must be set now so version history model can use same id template.archived = False + + template.template_redacted = TemplateRedacted( + template=template, + redact_personalisation=False, + updated_by=template.created_by + ) + db.session.add(template) @@ -25,6 +33,14 @@ def dao_update_template(template): db.session.add(template) +@transactional +def dao_redact_template(template, user_id): + template.template_redacted.redact_personalisation = True + template.template_redacted.updated_at = datetime.utcnow() + template.template_redacted.updated_by_id = user_id + db.session.add(template.template_redacted) + + def dao_get_template_by_id_and_service_id(template_id, service_id, version=None): if version is not None: return TemplateHistory.query.filter_by( diff --git a/app/models.py b/app/models.py index 7a4b9ef3b..8d6a64464 100644 --- a/app/models.py +++ b/app/models.py @@ -434,12 +434,16 @@ class Template(db.Model): subject = db.Column(db.Text, index=False, unique=False, nullable=True) created_by_id = db.Column(UUID(as_uuid=True), db.ForeignKey('users.id'), index=True, nullable=False) created_by = db.relationship('User') - version = db.Column(db.Integer, default=1, nullable=False) - process_type = db.Column(db.String(255), - db.ForeignKey('template_process_type.name'), - index=True, - nullable=False, - default=NORMAL) + version = db.Column(db.Integer, default=0, nullable=False) + process_type = db.Column( + db.String(255), + db.ForeignKey('template_process_type.name'), + index=True, + nullable=False, + default=NORMAL + ) + + redact_personalisation = association_proxy('template_redacted', 'redact_personalisation') def get_link(self): # TODO: use "/v2/" route once available @@ -465,6 +469,19 @@ class Template(db.Model): return serialized +class TemplateRedacted(db.Model): + __tablename__ = 'template_redacted' + + template_id = db.Column(UUID(as_uuid=True), db.ForeignKey('templates.id'), primary_key=True, nullable=False) + redact_personalisation = db.Column(db.Boolean, nullable=False, default=False) + updated_at = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow) + updated_by_id = db.Column(UUID(as_uuid=True), db.ForeignKey('users.id'), nullable=False) + updated_by = db.relationship('User') + + # uselist=False as this is a one-to-one relationship + template = db.relationship('Template', uselist=False, backref=db.backref('template_redacted', uselist=False)) + + class TemplateHistory(db.Model): __tablename__ = 'templates_history' diff --git a/app/schemas.py b/app/schemas.py index 34f057524..48151dba2 100644 --- a/app/schemas.py +++ b/app/schemas.py @@ -297,6 +297,11 @@ class BaseTemplateSchema(BaseSchema): strict = True +class TemplateRedactedSchema(BaseSchema): + class Meta: + model = models.TemplateRedacted + + class TemplateSchema(BaseTemplateSchema): created_by = field_for(models.Template, 'created_by', required=True) @@ -440,7 +445,7 @@ class NotificationWithTemplateSchema(BaseSchema): template = fields.Nested( TemplateSchema, - only=['id', 'version', 'name', 'template_type', 'content', 'subject'], + only=['id', 'version', 'name', 'template_type', 'content', 'subject', 'redact_personalisation'], dump_only=True ) job = fields.Nested(JobSchema, only=["id", "original_file_name"], dump_only=True) diff --git a/app/template/rest.py b/app/template/rest.py index acb2043dd..a9bd44f1f 100644 --- a/app/template/rest.py +++ b/app/template/rest.py @@ -8,6 +8,7 @@ from flask import ( from app.dao.templates_dao import ( dao_update_template, dao_create_template, + dao_redact_template, dao_get_template_by_id_and_service_id, dao_get_all_templates_for_service, dao_get_template_versions @@ -55,9 +56,15 @@ def create_template(service_id): def update_template(service_id, template_id): fetched_template = dao_get_template_by_id_and_service_id(template_id=template_id, service_id=service_id) + data = request.get_json() + + # if redacting, don't update anything else + if data.get('redact_personalisation') is True: + return redact_template(fetched_template, data) + current_data = dict(template_schema.dump(fetched_template).data.items()) updated_template = dict(template_schema.dump(fetched_template).data.items()) - updated_template.update(request.get_json()) + updated_template.update(data) # Check if there is a change to make. if _template_has_not_changed(current_data, updated_template): return jsonify(data=updated_template), 200 @@ -132,3 +139,16 @@ def _template_has_not_changed(current_data, updated_template): current_data[key] == updated_template[key] for key in ('name', 'content', 'subject', 'archived', 'process_type') ) + + +def redact_template(template, data): + # we also don't need to check what was passed in redact_personalisation - its presence in the dict is enough. + if 'created_by' not in data: + message = 'Field is required' + errors = {'created_by': [message]} + raise InvalidRequest(errors, status_code=400) + + # if it's already redacted, then just return 200 straight away. + if not template.redact_personalisation: + dao_redact_template(template, data['created_by']) + return 'null', 200 diff --git a/migrations/versions/0102_template_redacted.py b/migrations/versions/0102_template_redacted.py new file mode 100644 index 000000000..16d670d39 --- /dev/null +++ b/migrations/versions/0102_template_redacted.py @@ -0,0 +1,31 @@ +"""empty message + +Revision ID: db6d9d9f06bc +Revises: 0101_een_logo +Create Date: 2017-06-27 15:37:28.878359 + +""" + +# revision identifiers, used by Alembic. +revision = 'db6d9d9f06bc' +down_revision = '0101_een_logo' + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +def upgrade(): + op.create_table('template_redacted', + sa.Column('template_id', postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('redact_personalisation', sa.Boolean(), nullable=False), + sa.Column('updated_at', sa.DateTime(), nullable=False), + sa.Column('updated_by_id', postgresql.UUID(as_uuid=True), nullable=False), + sa.ForeignKeyConstraint(['template_id'], ['templates.id'], ), + sa.ForeignKeyConstraint(['updated_by_id'], ['users.id'], ), + sa.PrimaryKeyConstraint('template_id') + ) + op.create_index(op.f('ix_template_redacted_updated_by_id'), 'template_redacted', ['updated_by_id'], unique=False) + + +def downgrade(): + op.drop_table('template_redacted') diff --git a/migrations/versions/0103_add_historical_redact.py b/migrations/versions/0103_add_historical_redact.py new file mode 100644 index 000000000..8d073bbd3 --- /dev/null +++ b/migrations/versions/0103_add_historical_redact.py @@ -0,0 +1,43 @@ +"""empty message + +Revision ID: 0103_add_historical_redact +Revises: db6d9d9f06bc +Create Date: 2017-06-29 12:44:16.815039 + +""" + +# revision identifiers, used by Alembic. +revision = '0103_add_historical_redact' +down_revision = 'db6d9d9f06bc' + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql +from flask import current_app + +def upgrade(): + op.execute( + """ + INSERT INTO template_redacted + ( + template_id, + redact_personalisation, + updated_at, + updated_by_id + ) + SELECT + templates.id, + false, + now(), + '{notify_user}' + FROM + templates + LEFT JOIN template_redacted on template_redacted.template_id = templates.id + WHERE template_redacted.template_id IS NULL + """.format(notify_user=current_app.config['NOTIFY_USER_ID']) + ) + + +def downgrade(): + # data migration, no downloads + pass diff --git a/tests/app/dao/test_templates_dao.py b/tests/app/dao/test_templates_dao.py index 182f91928..58ad14e15 100644 --- a/tests/app/dao/test_templates_dao.py +++ b/tests/app/dao/test_templates_dao.py @@ -1,14 +1,21 @@ +from datetime import datetime + +from freezegun import freeze_time from sqlalchemy.orm.exc import NoResultFound +import pytest + from app.dao.templates_dao import ( dao_create_template, dao_get_template_by_id_and_service_id, dao_get_all_templates_for_service, dao_update_template, dao_get_template_versions, - dao_get_templates_for_cache) + dao_get_templates_for_cache, + dao_redact_template) +from app.models import Template, TemplateHistory, TemplateRedacted + from tests.app.conftest import sample_template as create_sample_template -from app.models import Template, TemplateHistory -import pytest +from tests.app.db import create_template @pytest.mark.parametrize('template_type, subject', [ @@ -35,6 +42,17 @@ def test_create_template(sample_service, sample_user, template_type, subject): assert dao_get_all_templates_for_service(sample_service.id)[0].process_type == 'normal' +def test_create_template_creates_redact_entry(sample_service): + assert TemplateRedacted.query.count() == 0 + + template = create_template(sample_service) + + redacted = TemplateRedacted.query.one() + assert redacted.template_id == template.id + assert redacted.redact_personalisation is False + assert redacted.updated_by_id == sample_service.created_by_id + + def test_update_template(sample_service, sample_user): data = { 'name': 'Sample Template', @@ -53,6 +71,20 @@ def test_update_template(sample_service, sample_user): assert dao_get_all_templates_for_service(sample_service.id)[0].name == 'new name' +def test_redact_template(sample_template): + redacted = TemplateRedacted.query.one() + assert redacted.template_id == sample_template.id + assert redacted.redact_personalisation is False + + time = datetime.now() + with freeze_time(time): + dao_redact_template(sample_template, sample_template.created_by_id) + + assert redacted.redact_personalisation is True + assert redacted.updated_at == time + assert redacted.updated_by_id == sample_template.created_by_id + + def test_get_all_templates_for_service(notify_db, notify_db_session, service_factory): service_1 = service_factory.get('service 1', email_from='service.1') service_2 = service_factory.get('service 2', email_from='service.2') diff --git a/tests/app/service/test_rest.py b/tests/app/service/test_rest.py index 26fa9034e..de24830f0 100644 --- a/tests/app/service/test_rest.py +++ b/tests/app/service/test_rest.py @@ -11,6 +11,7 @@ from freezegun import freeze_time from app import encryption from app.dao.users_dao import save_model_user from app.dao.services_dao import dao_remove_user_from_service +from app.dao.templates_dao import dao_redact_template from app.models import ( User, Organisation, Rate, Service, ServicePermission, Notification, DVLA_ORG_LAND_REGISTRY, @@ -19,7 +20,7 @@ from app.models import ( ) from tests import create_authorization_header -from tests.app.db import create_template, create_service_inbound_api, create_user +from tests.app.db import create_template, create_service_inbound_api, create_user, create_notification from tests.app.conftest import ( sample_service as create_service, sample_user_service_permission as create_user_service_permission, @@ -2263,3 +2264,37 @@ def test_send_one_off_notification(admin_request, sample_template, mocker): noti = Notification.query.one() assert response['id'] == str(noti.id) + + +def test_get_notification_for_service_includes_template_redacted(admin_request, sample_notification): + resp = admin_request.get( + 'service.get_notification_for_service', + service_id=sample_notification.service_id, + notification_id=sample_notification.id + ) + + assert resp['id'] == str(sample_notification.id) + assert resp['template']['redact_personalisation'] is False + + +def test_get_all_notifications_for_service_includes_template_redacted(admin_request, sample_service): + normal_template = create_template(sample_service) + + redacted_template = create_template(sample_service) + dao_redact_template(redacted_template, sample_service.created_by_id) + + with freeze_time('2000-01-01'): + redacted_noti = create_notification(redacted_template) + with freeze_time('2000-01-02'): + normal_noti = create_notification(normal_template) + + resp = admin_request.get( + 'service.get_all_notifications_for_service', + service_id=sample_service.id + ) + + assert resp['notifications'][0]['id'] == str(normal_noti.id) + assert resp['notifications'][0]['template']['redact_personalisation'] is False + + assert resp['notifications'][1]['id'] == str(redacted_noti.id) + assert resp['notifications'][1]['template']['redact_personalisation'] is True diff --git a/tests/app/template/test_rest.py b/tests/app/template/test_rest.py index 13e21dcfd..dfb773453 100644 --- a/tests/app/template/test_rest.py +++ b/tests/app/template/test_rest.py @@ -1,11 +1,17 @@ +import uuid import json import random import string +from datetime import datetime, timedelta + import pytest +from freezegun import freeze_time + from app.models import Template +from app.dao.templates_dao import dao_get_template_by_id, dao_redact_template + from tests import create_authorization_header from tests.app.conftest import sample_template as create_sample_template -from app.dao.templates_dao import dao_get_template_by_id @pytest.mark.parametrize('template_type, subject', [ @@ -539,3 +545,92 @@ def test_update_set_process_type_on_template(client, sample_template): template = dao_get_template_by_id(sample_template.id) assert template.process_type == 'priority' + + +def test_update_redact_template(admin_request, sample_template): + assert sample_template.redact_personalisation is False + + data = { + 'redact_personalisation': True, + 'created_by': str(sample_template.created_by_id) + } + + dt = datetime.now() + + with freeze_time(dt): + resp = admin_request.post( + 'template.update_template', + service_id=sample_template.service_id, + template_id=sample_template.id, + _data=data + ) + + assert resp is None + + assert sample_template.redact_personalisation is True + assert sample_template.template_redacted.updated_by_id == sample_template.created_by_id + assert sample_template.template_redacted.updated_at == dt + + assert sample_template.version == 1 + + +def test_update_redact_template_ignores_other_properties(admin_request, sample_template): + data = { + 'name': 'Foo', + 'redact_personalisation': True, + 'created_by': str(sample_template.created_by_id) + } + + admin_request.post( + 'template.update_template', + service_id=sample_template.service_id, + template_id=sample_template.id, + _data=data + ) + + assert sample_template.redact_personalisation is True + assert sample_template.name != 'Foo' + + +def test_update_redact_template_does_nothing_if_already_redacted(admin_request, sample_template): + dt = datetime.now() + with freeze_time(dt): + dao_redact_template(sample_template, sample_template.created_by_id) + + data = { + 'redact_personalisation': True, + 'created_by': str(sample_template.created_by_id) + } + + with freeze_time(dt + timedelta(days=1)): + resp = admin_request.post( + 'template.update_template', + service_id=sample_template.service_id, + template_id=sample_template.id, + _data=data + ) + + assert resp is None + + assert sample_template.redact_personalisation is True + # make sure that it hasn't been updated + assert sample_template.template_redacted.updated_at == dt + + +def test_update_redact_template_400s_if_no_created_by(admin_request, sample_template): + original_updated_time = sample_template.template_redacted.updated_at + resp = admin_request.post( + 'template.update_template', + service_id=sample_template.service_id, + template_id=sample_template.id, + _data={'redact_personalisation': True}, + _expected_status=400 + ) + + assert resp == { + 'result': 'error', + 'message': {'created_by': ['Field is required']} + } + + assert sample_template.redact_personalisation is False + assert sample_template.template_redacted.updated_at == original_updated_time