Merge branch 'master' into secure-endpoints

Conflicts:
	app/__init__.py

tests/app/celery/test_tasks.py

@@ -1,8 +1,9 @@
 import uuid
 from datetime import datetime
-from unittest.mock import Mock, ANY, call
+from unittest.mock import Mock
 
 import pytest
+from flask import current_app
 from freezegun import freeze_time
 from sqlalchemy.exc import SQLAlchemyError
 from notifications_utils.template import SMSMessageTemplate, WithSubjectTemplate
@@ -11,7 +12,7 @@ from celery.exceptions import Retry
 from app import (encryption, DATETIME_FORMAT)
 from app.celery import provider_tasks
 from app.celery import tasks
-from app.celery.tasks import s3
+from app.celery.tasks import s3, build_dvla_file
 from app.celery.tasks import (
     process_job,
     process_row,
@@ -31,7 +32,7 @@ from tests.app.conftest import (
     sample_email_template,
     sample_notification
 )
-from tests.app.db import create_user
+from tests.app.db import create_user, create_notification, create_job
 
 
 class AnyStringWith(str):
@@ -73,7 +74,9 @@ def test_should_process_sms_job(sample_job, mocker):
     mocker.patch('app.celery.tasks.s3.get_job_from_s3', return_value=load_example_csv('sms'))
     mocker.patch('app.celery.tasks.send_sms.apply_async')
     mocker.patch('app.encryption.encrypt', return_value="something_encrypted")
+    mocker.patch('app.celery.tasks.build_dvla_file')
     mocker.patch('app.celery.tasks.create_uuid', return_value="uuid")
+    mocker.patch('app.celery.tasks.build_dvla_file')
 
     process_job(sample_job.id)
     s3.get_job_from_s3.assert_called_once_with(
@@ -94,6 +97,7 @@ def test_should_process_sms_job(sample_job, mocker):
     )
     job = jobs_dao.dao_get_job_by_id(sample_job.id)
     assert job.job_status == 'finished'
+    tasks.build_dvla_file.assert_not_called()
 
 
 @freeze_time("2016-01-01 11:09:00.061258")
@@ -105,6 +109,7 @@ def test_should_not_process_sms_job_if_would_exceed_send_limits(notify_db,
 
     mocker.patch('app.celery.tasks.s3.get_job_from_s3', return_value=load_example_csv('multiple_sms'))
     mocker.patch('app.celery.tasks.process_row')
+    mocker.patch('app.celery.tasks.build_dvla_file')
 
     process_job(job.id)
 
@@ -112,6 +117,7 @@ def test_should_not_process_sms_job_if_would_exceed_send_limits(notify_db,
     assert job.job_status == 'sending limits exceeded'
     assert s3.get_job_from_s3.called is False
     assert tasks.process_row.called is False
+    tasks.build_dvla_file.assert_not_called()
 
 
 def test_should_not_process_sms_job_if_would_exceed_send_limits_inc_today(notify_db,
@@ -124,6 +130,7 @@ def test_should_not_process_sms_job_if_would_exceed_send_limits_inc_today(notify
 
     mocker.patch('app.celery.tasks.s3.get_job_from_s3', return_value=load_example_csv('sms'))
     mocker.patch('app.celery.tasks.process_row')
+    mocker.patch('app.celery.tasks.build_dvla_file')
 
     process_job(job.id)
 
@@ -131,6 +138,7 @@ def test_should_not_process_sms_job_if_would_exceed_send_limits_inc_today(notify
     assert job.job_status == 'sending limits exceeded'
     assert s3.get_job_from_s3.called is False
     assert tasks.process_row.called is False
+    tasks.build_dvla_file.assert_not_called()
 
 
 def test_should_not_process_email_job_if_would_exceed_send_limits_inc_today(notify_db, notify_db_session, mocker):
@@ -142,6 +150,7 @@ def test_should_not_process_email_job_if_would_exceed_send_limits_inc_today(noti
 
     mocker.patch('app.celery.tasks.s3.get_job_from_s3')
     mocker.patch('app.celery.tasks.process_row')
+    mocker.patch('app.celery.tasks.build_dvla_file')
 
     process_job(job.id)
 
@@ -149,6 +158,7 @@ def test_should_not_process_email_job_if_would_exceed_send_limits_inc_today(noti
     assert job.job_status == 'sending limits exceeded'
     assert s3.get_job_from_s3.called is False
     assert tasks.process_row.called is False
+    tasks.build_dvla_file.assert_not_called()
 
 
 @freeze_time("2016-01-01 11:09:00.061258")
@@ -159,6 +169,7 @@ def test_should_not_process_email_job_if_would_exceed_send_limits(notify_db, not
 
     mocker.patch('app.celery.tasks.s3.get_job_from_s3')
     mocker.patch('app.celery.tasks.process_row')
+    mocker.patch('app.celery.tasks.build_dvla_file')
 
     process_job(job.id)
 
@@ -166,6 +177,7 @@ def test_should_not_process_email_job_if_would_exceed_send_limits(notify_db, not
     assert job.job_status == 'sending limits exceeded'
     assert s3.get_job_from_s3.called is False
     assert tasks.process_row.called is False
+    tasks.build_dvla_file.assert_not_called()
 
 
 def test_should_not_process_job_if_already_pending(notify_db, notify_db_session, mocker):
@@ -173,11 +185,13 @@ def test_should_not_process_job_if_already_pending(notify_db, notify_db_session,
 
     mocker.patch('app.celery.tasks.s3.get_job_from_s3')
     mocker.patch('app.celery.tasks.process_row')
+    mocker.patch('app.celery.tasks.build_dvla_file')
 
     process_job(job.id)
 
     assert s3.get_job_from_s3.called is False
     assert tasks.process_row.called is False
+    tasks.build_dvla_file.assert_not_called()
 
 
 @freeze_time("2016-01-01 11:09:00.061258")
@@ -269,6 +283,7 @@ def test_should_process_letter_job(sample_letter_job, mocker):
     mocker.patch('app.celery.tasks.send_email.apply_async')
     process_row_mock = mocker.patch('app.celery.tasks.process_row')
     mocker.patch('app.celery.tasks.create_uuid', return_value="uuid")
+    mocker.patch('app.celery.tasks.build_dvla_file')
 
     process_job(sample_letter_job.id)
 
@@ -294,6 +309,7 @@ def test_should_process_letter_job(sample_letter_job, mocker):
     assert process_row_mock.call_count == 1
 
     assert sample_letter_job.job_status == 'finished'
+    tasks.build_dvla_file.apply_async.assert_called_once_with([str(sample_letter_job.id)], queue="process-job")
 
 
 def test_should_process_all_sms_job(sample_job,
@@ -930,6 +946,7 @@ def test_should_cancel_job_if_service_is_inactive(sample_service,
 
     mocker.patch('app.celery.tasks.s3.get_job_from_s3')
     mocker.patch('app.celery.tasks.process_row')
+    mock_dvla_file_task = mocker.patch('app.celery.tasks.build_dvla_file')
 
     process_job(sample_job.id)
 
@@ -937,6 +954,7 @@ def test_should_cancel_job_if_service_is_inactive(sample_service,
     assert job.job_status == 'cancelled'
     s3.get_job_from_s3.assert_not_called()
     tasks.process_row.assert_not_called()
+    mock_dvla_file_task.assert_not_called()
 
 
 @pytest.mark.parametrize('template_type, expected_class', [
@@ -946,3 +964,35 @@ def test_should_cancel_job_if_service_is_inactive(sample_service,
 ])
 def test_get_template_class(template_type, expected_class):
     assert get_template_class(template_type) == expected_class
+
+
+def test_build_dvla_file(sample_letter_template, mocker):
+    job = create_job(template=sample_letter_template, notification_count=2)
+    create_notification(template=job.template, job=job)
+    create_notification(template=job.template, job=job)
+
+    mocked = mocker.patch("app.celery.tasks.s3upload")
+    mocker.patch("app.celery.tasks.LetterDVLATemplate.__str__", return_value="dvla|string")
+    build_dvla_file(job.id)
+
+    file = "dvla|string\ndvla|string\n"
+
+    assert mocked.called
+    mocked.assert_called_once_with(filedata=file,
+                                   region=current_app.config['AWS_REGION'],
+                                   bucket_name=current_app.config['DVLA_UPLOAD_BUCKET_NAME'],
+                                   file_location="{}-dvla-job.text".format(job.id))
+
+
+def test_build_dvla_file_retries_if_all_notifications_are_not_created(sample_letter_template, mocker):
+    job = create_job(template=sample_letter_template, notification_count=2)
+    create_notification(template=job.template, job=job)
+
+    mocked = mocker.patch("app.celery.tasks.s3upload")
+    mocker.patch('app.celery.tasks.build_dvla_file.retry', side_effect=Retry)
+    with pytest.raises(Retry):
+        build_dvla_file(job.id)
+    mocked.assert_not_called()
+
+    tasks.build_dvla_file.retry.assert_called_with(queue='retry',
+                                                   exc="All notifications for job {} are not persisted".format(job.id))

tests/app/dao/test_jobs_dao.py

@@ -12,8 +12,9 @@ from app.dao.jobs_dao import (
     dao_set_scheduled_jobs_to_pending,
     dao_get_future_scheduled_job_by_id_and_service_id,
     dao_get_notification_outcomes_for_job,
-    dao_get_jobs_older_than
-)
+    dao_get_jobs_older_than,
+    all_notifications_are_created_for_job,
+    dao_get_all_notifications_for_job)
 from app.models import Job
 
 from tests.app.conftest import sample_notification as create_notification
@@ -314,3 +315,30 @@ def test_get_jobs_for_service_doesnt_return_test_messages(notify_db, notify_db_s
     jobs = dao_get_jobs_by_service_id(sample_job.service_id).items
 
     assert jobs == [sample_job]
+
+
+def test_all_notifications_are_created_for_job_returns_true(notify_db, notify_db_session):
+    job = create_job(notify_db=notify_db, notify_db_session=notify_db_session, notification_count=2)
+    create_notification(notify_db=notify_db, notify_db_session=notify_db_session, job=job)
+    create_notification(notify_db=notify_db, notify_db_session=notify_db_session, job=job)
+    job_is_complete = all_notifications_are_created_for_job(job.id)
+    assert job_is_complete
+
+
+def test_all_notifications_are_created_for_job_returns_false(notify_db, notify_db_session):
+    job = create_job(notify_db=notify_db, notify_db_session=notify_db_session, notification_count=2)
+    job_is_complete = all_notifications_are_created_for_job(job.id)
+    assert not job_is_complete
+
+
+def test_are_all_notifications_created_for_job_returns_false_when_job_does_not_exist():
+    job_is_complete = all_notifications_are_created_for_job(uuid.uuid4())
+    assert not job_is_complete
+
+
+def test_dao_get_all_notifications_for_job(notify_db, notify_db_session, sample_job):
+    create_notification(notify_db=notify_db, notify_db_session=notify_db_session, job=sample_job)
+    create_notification(notify_db=notify_db, notify_db_session=notify_db_session, job=sample_job)
+    create_notification(notify_db=notify_db, notify_db_session=notify_db_session, job=sample_job)
+
+    assert len(dao_get_all_notifications_for_job(sample_job.id)) == 3

tests/app/db.py

@@ -1,7 +1,8 @@
 from datetime import datetime
 import uuid
 
-from app.models import Service, User, Template, Notification, SMS_TYPE, KEY_TYPE_NORMAL
+from app.dao.jobs_dao import dao_create_job
+from app.models import Service, User, Template, Notification, SMS_TYPE, KEY_TYPE_NORMAL, Job
 from app.dao.users_dao import save_model_user
 from app.dao.notifications_dao import dao_create_notification
 from app.dao.templates_dao import dao_create_template
@@ -105,3 +106,30 @@ def create_notification(
     notification = Notification(**data)
     dao_create_notification(notification)
     return notification
+
+
+def create_job(template,
+               notification_count=1,
+               created_at=None,
+               job_status='pending',
+               scheduled_for=None,
+               processing_started=None,
+               original_file_name='some.csv'):
+
+    data = {
+        'id': uuid.uuid4(),
+        'service_id': template.service_id,
+        'service': template.service,
+        'template_id': template.id,
+        'template_version': template.version,
+        'original_file_name': original_file_name,
+        'notification_count': notification_count,
+        'created_at': created_at or datetime.utcnow(),
+        'created_by': template.created_by,
+        'job_status': job_status,
+        'scheduled_for': scheduled_for,
+        'processing_started': processing_started
+    }
+    job = Job(**data)
+    dao_create_job(job)
+    return job

tests/app/v2/template/test_get_template.py

@@ -0,0 +1,93 @@
+import pytest
+import uuid
+
+from flask import json
+
+from app import DATETIME_FORMAT
+from app.models import EMAIL_TYPE, SMS_TYPE, LETTER_TYPE
+from tests import create_authorization_header
+from tests.app.db import create_template
+
+template_types = [EMAIL_TYPE, SMS_TYPE, LETTER_TYPE]
+valid_version_params = [None, 1]
+
+
+@pytest.mark.parametrize("tmp_type", template_types)
+@pytest.mark.parametrize("version", valid_version_params)
+def test_get_email_template_by_id_returns_200(client, sample_service, tmp_type, version):
+    template = create_template(sample_service, template_type=tmp_type)
+    auth_header = create_authorization_header(service_id=sample_service.id)
+
+    version_path = '/version/{}'.format(version) if version else ''
+
+    response = client.get(path='/v2/template/{}{}'.format(template.id, version_path),
+                          headers=[('Content-Type', 'application/json'), auth_header])
+
+    assert response.status_code == 200
+    assert response.headers['Content-type'] == 'application/json'
+
+    json_response = json.loads(response.get_data(as_text=True))
+
+    expected_response = {
+        'id': '{}'.format(template.id),
+        'type': '{}'.format(template.template_type),
+        'created_at': template.created_at.strftime(DATETIME_FORMAT),
+        'updated_at': None,
+        'version': template.version,
+        'created_by': template.created_by.email_address,
+        'body': template.content,
+        "subject": template.subject if tmp_type == EMAIL_TYPE else None
+    }
+
+    assert json_response == expected_response
+
+
+def test_get_template_with_non_existent_template_id_returns_404(client, sample_service):
+    auth_header = create_authorization_header(service_id=sample_service.id)
+
+    random_template_id = str(uuid.uuid4())
+
+    response = client.get(path='/v2/template/{}'.format(random_template_id),
+                          headers=[('Content-Type', 'application/json'), auth_header])
+
+    assert response.status_code == 404
+    assert response.headers['Content-type'] == 'application/json'
+
+    json_response = json.loads(response.get_data(as_text=True))
+
+    assert json_response == {
+        "errors": [
+            {
+                "error": "NoResultFound",
+                "message": "No result found"
+            }
+        ],
+        "status_code": 404
+    }
+
+
+@pytest.mark.parametrize("tmp_type", template_types)
+def test_get_template_with_non_existent_version_returns_404(client, sample_service, tmp_type):
+    template = create_template(sample_service, template_type=tmp_type)
+
+    auth_header = create_authorization_header(service_id=sample_service.id)
+
+    invalid_version = template.version + 1
+
+    response = client.get(path='/v2/template/{}/version/{}'.format(template.id, invalid_version),
+                          headers=[('Content-Type', 'application/json'), auth_header])
+
+    assert response.status_code == 404
+    assert response.headers['Content-type'] == 'application/json'
+
+    json_response = json.loads(response.get_data(as_text=True))
+
+    assert json_response == {
+        "errors": [
+            {
+                "error": "NoResultFound",
+                "message": "No result found"
+            }
+        ],
+        "status_code": 404
+    }

tests/app/v2/template/test_template_schemas.py

@@ -0,0 +1,70 @@
+import uuid
+
+import pytest
+from flask import json
+
+from app.v2.template.template_schemas import (
+    get_template_by_id_response,
+    get_template_by_id_request
+)
+from app.schema_validation import validate
+from jsonschema.exceptions import ValidationError
+
+
+valid_json = {
+    'id': str(uuid.uuid4()),
+    'type': 'email',
+    'created_at': '2017-01-10T18:25:43.511Z',
+    'updated_at': None,
+    'version': 1,
+    'created_by': 'someone@test.com',
+    'body': "some body"
+}
+
+valid_json_with_optionals = {
+    'id': str(uuid.uuid4()),
+    'type': 'email',
+    'created_at': '2017-01-10T18:25:43.511Z',
+    'updated_at': None,
+    'version': 1,
+    'created_by': 'someone',
+    'body': "some body",
+    'subject': "some subject"
+}
+
+valid_request_args = [
+    {"id": str(uuid.uuid4()), "version": 1}, {"id": str(uuid.uuid4())}]
+
+invalid_request_args = [
+    ({"id": str(uuid.uuid4()), "version": "test"}, ["version test is not of type integer, null"]),
+    ({"id": str(uuid.uuid4()), "version": 0}, ["version 0 is less than the minimum of 1"]),
+    ({"version": 1}, ["id is a required property"]),
+    ({"id": "invalid_uuid"}, ["id is not a valid UUID"]),
+    ({"id": "invalid_uuid", "version": 0}, ["version 0 is less than the minimum of 1", "id is not a valid UUID"])
+]
+
+
+@pytest.mark.parametrize("args", valid_request_args)
+def test_get_template_request_schema_against_valid_args_is_valid(args):
+    assert validate(args, get_template_by_id_request) == args
+
+
+@pytest.mark.parametrize("args,error_message", invalid_request_args)
+def test_get_template_request_schema_against_invalid_args_is_invalid(args, error_message):
+    with pytest.raises(ValidationError) as e:
+        validate(args, get_template_by_id_request)
+    errors = json.loads(str(e.value))
+
+    assert errors['status_code'] == 400
+
+    for error in errors['errors']:
+        assert error['message'] in error_message
+
+
+@pytest.mark.parametrize("response", [valid_json, valid_json_with_optionals])
+@pytest.mark.parametrize("updated_datetime", [None, '2017-01-11T18:25:43.511Z'])
+def test_get_template_response_schema_is_valid(response, updated_datetime):
+    if updated_datetime:
+        response['updated_at'] = updated_datetime
+
+    assert validate(response, get_template_by_id_response) == response