Merge branch 'master' into vb-remove-ip-whitelist

tests/app/celery/test_scheduled_tasks.py

@@ -24,6 +24,7 @@ from app.celery.scheduled_tasks import (
     remove_transformed_dvla_files,
     run_scheduled_jobs,
     run_letter_jobs,
+    trigger_letter_pdfs_for_day,
     run_letter_api_notifications,
     populate_monthly_billing,
     s3,
@@ -43,6 +44,7 @@ from app.dao.provider_details_dao import (
     dao_update_provider_details,
     get_current_provider
 )
+from app.dao.service_permissions_dao import dao_add_service_permission
 from app.models import (
     MonthlyBilling,
     NotificationHistory,
@@ -713,6 +715,36 @@ def test_run_letter_jobs(client, mocker, sample_letter_template):
                                         queue=QueueNames.PROCESS_FTP)
 
 
+@freeze_time("2017-12-18 17:50")
+def test_trigger_letter_pdfs_for_day(client, mocker, sample_letter_template):
+    dao_add_service_permission(sample_letter_template.service.id, 'letters_as_pdf')
+
+    create_notification(template=sample_letter_template, created_at='2017-12-17 17:30:00')
+    create_notification(template=sample_letter_template, created_at='2017-12-18 17:29:59')
+
+    mock_celery = mocker.patch("app.celery.tasks.notify_celery.send_task")
+
+    trigger_letter_pdfs_for_day()
+
+    mock_celery.assert_called_once_with(name='collate-letter-pdfs-for-day',
+                                        args=('2017-12-18',),
+                                        queue=QueueNames.LETTERS)
+
+
+@freeze_time("2017-12-18 17:50")
+def test_trigger_letter_pdfs_for_day_send_task_not_called_if_no_notis_for_day(
+        client, mocker, sample_letter_template):
+    dao_add_service_permission(sample_letter_template.service.id, 'letters_as_pdf')
+
+    create_notification(template=sample_letter_template, created_at='2017-12-15 17:30:00')
+
+    mock_celery = mocker.patch("app.celery.tasks.notify_celery.send_task")
+
+    trigger_letter_pdfs_for_day()
+
+    assert not mock_celery.called
+
+
 def test_run_letter_jobs_does_nothing_if_no_ready_jobs(client, mocker, sample_letter_template):
     create_job(sample_letter_template, job_status=JOB_STATUS_IN_PROGRESS)
     create_job(sample_letter_template, job_status=JOB_STATUS_SENT_TO_DVLA)

tests/app/celery/test_tasks.py

@@ -1040,6 +1040,47 @@ def test_save_letter_saves_letter_to_database(mocker, notify_db_session):
     assert notification_db.reply_to_text == "Address contact"
 
 
+def test_save_letter_uses_template_reply_to_text(mocker, notify_db_session):
+    service = create_service()
+    create_letter_contact(service=service, contact_block="Address contact", is_default=True)
+    template_contact = create_letter_contact(
+        service=service,
+        contact_block="Template address contact",
+        is_default=False
+    )
+    template = create_template(
+        service=service,
+        template_type=LETTER_TYPE,
+        reply_to=template_contact.id
+    )
+
+    job = create_job(template=template)
+
+    mocker.patch('app.celery.tasks.create_random_identifier', return_value="this-is-random-in-real-life")
+
+    personalisation = {
+        'addressline1': 'Foo',
+        'addressline2': 'Bar',
+        'postcode': 'Flob',
+    }
+    notification_json = _notification_json(
+        template=job.template,
+        to='Foo',
+        personalisation=personalisation,
+        job_id=job.id,
+        row_number=1
+    )
+
+    save_letter(
+        job.service_id,
+        uuid.uuid4(),
+        encryption.encrypt(notification_json),
+    )
+
+    notification_db = Notification.query.one()
+    assert notification_db.reply_to_text == "Template address contact"
+
+
 def test_save_letter_calls_update_noti_to_sent_task_with_letters_as_pdf_permission_in_research_mode(
         mocker, notify_db_session, sample_letter_job):
     sample_letter_job.service.research_mode = True

tests/app/dao/notification_dao/test_notification_dao.py

@@ -10,6 +10,7 @@ from app.dao.notifications_dao import (
     dao_create_notification,
     dao_created_scheduled_notification,
     dao_delete_notifications_and_history_by_id,
+    dao_get_count_of_letters_to_process_for_date,
     dao_get_last_notification_added_for_job_id,
     dao_get_last_template_usage,
     dao_get_notifications_by_to_field,
@@ -33,6 +34,7 @@ from app.dao.notifications_dao import (
     dao_get_notifications_by_references
 )
 from app.dao.services_dao import dao_update_service
+from app.dao.service_permissions_dao import dao_add_service_permission
 from app.models import (
     Job,
     Notification,
@@ -2001,3 +2003,90 @@ def test_dao_get_notifications_by_reference(sample_template):
     assert len(notifications) == 2
     assert notifications[0].id in [notification_1.id, notification_2.id]
     assert notifications[1].id in [notification_1.id, notification_2.id]
+
+
+@freeze_time("2017-12-18 17:50")
+def test_dao_get_count_of_letters_to_process_for_today(sample_letter_template):
+    dao_add_service_permission(sample_letter_template.service.id, 'letters_as_pdf')
+
+    # expected
+    create_notification(template=sample_letter_template, created_at='2017-12-17 17:30:00')
+    create_notification(template=sample_letter_template, created_at='2017-12-18 17:29:59')
+
+    # not expected
+    create_notification(template=sample_letter_template, created_at='2017-12-17 17:29:59')
+    create_notification(template=sample_letter_template, created_at='2017-12-18 17:30:00')
+
+    count_for_date = dao_get_count_of_letters_to_process_for_date()
+
+    assert count_for_date == 2
+
+
+@freeze_time("2017-12-18 17:50")
+def test_dao_get_count_of_letters_to_process_for_date_in_past(sample_letter_template):
+    dao_add_service_permission(sample_letter_template.service.id, 'letters_as_pdf')
+
+    # expected
+    create_notification(template=sample_letter_template, created_at='2017-12-15 17:29:59')
+
+    # not expected
+    create_notification(template=sample_letter_template, created_at='2017-12-15 17:30:00')
+    create_notification(template=sample_letter_template, created_at='2017-12-18 17:29:00')
+
+    count_for_date = dao_get_count_of_letters_to_process_for_date(date(2017, 12, 15))
+
+    assert count_for_date == 1
+
+
+@freeze_time("2017-12-18 17:50")
+def test_dao_get_count_of_letters_to_process_for_date_in_future_does_not_raise_error(sample_letter_template):
+    dao_add_service_permission(sample_letter_template.service.id, 'letters_as_pdf')
+
+    # not expected
+    create_notification(template=sample_letter_template, created_at='2017-12-18 17:30:00')
+    create_notification(template=sample_letter_template, created_at='2017-12-19 17:29:59')
+
+    count_for_date = dao_get_count_of_letters_to_process_for_date(date(2017, 12, 20))
+
+    assert count_for_date == 0
+
+
+def test_dao_get_count_of_letters_to_process_for_today_without_notis_does_not_raise_error(sample_letter_template):
+    count_for_date = dao_get_count_of_letters_to_process_for_date()
+
+    assert count_for_date == 0
+
+
+@freeze_time("2017-12-18 17:50")
+def test_dao_get_count_of_letters_to_process_for_date_ignores_service_not_letters_as_pdf(
+        sample_letter_template, sample_template):
+    # not expected
+    create_notification(template=sample_letter_template, created_at='2017-12-18 17:29:00')
+
+    dao_add_service_permission(sample_template.service.id, 'letters_as_pdf')
+    sample_template.template_type = 'letter'
+
+    # expected
+    create_notification(template=sample_template, created_at='2017-12-18 17:29:00')
+    create_notification(template=sample_template, created_at='2017-12-18 17:29:10')
+
+    count_for_date = dao_get_count_of_letters_to_process_for_date()
+
+    assert 'letters_as_pdf' not in [p.permission for p in sample_letter_template.service.permissions]
+    assert count_for_date == 2
+
+
+@freeze_time("2017-12-18 17:50")
+def test_dao_get_count_of_letters_to_process_for_date_ignores_test_keys(sample_letter_template):
+    dao_add_service_permission(sample_letter_template.service.id, 'letters_as_pdf')
+
+    # not expected
+    create_notification(template=sample_letter_template, key_type=KEY_TYPE_TEST, created_at='2017-12-18 17:29:00')
+
+    # expected
+    create_notification(template=sample_letter_template, created_at='2017-12-18 17:29:10')
+    create_notification(template=sample_letter_template, created_at='2017-12-18 17:29:20')
+
+    count_for_date = dao_get_count_of_letters_to_process_for_date()
+
+    assert count_for_date == 2

tests/app/db.py

@@ -123,7 +123,8 @@ def create_template(
     template_name=None,
     subject='Template subject',
     content='Dear Sir/Madam, Hello. Yours Truly, The Government.',
-    template_id=None
+    template_id=None,
+    reply_to=None
 ):
     data = {
         'name': template_name or '{} Template Name'.format(template_type),
@@ -131,6 +132,7 @@ def create_template(
         'content': content,
         'service': service,
         'created_by': service.created_by,
+        'reply_to': reply_to,
     }
     if template_type != SMS_TYPE:
         data['subject'] = subject

tests/app/notifications/test_validators.py

@@ -3,7 +3,7 @@ from freezegun import freeze_time
 from flask import current_app
 
 import app
-from app.models import INTERNATIONAL_SMS_TYPE, SMS_TYPE, EMAIL_TYPE
+from app.models import INTERNATIONAL_SMS_TYPE, SMS_TYPE, EMAIL_TYPE, LETTER_TYPE
 from app.notifications.validators import (
     check_service_over_daily_message_limit,
     check_template_is_for_notification_type,
@@ -13,7 +13,10 @@ from app.notifications.validators import (
     check_service_over_api_rate_limit,
     validate_and_format_recipient,
     check_service_email_reply_to_id,
-    check_service_sms_sender_id)
+    check_service_sms_sender_id,
+    check_service_letter_contact_id,
+    check_reply_to,
+)
 
 from app.v2.errors import (
     BadRequestError,
@@ -26,7 +29,7 @@ from tests.app.conftest import (
     sample_service as create_service,
     sample_service_whitelist,
     sample_api_key)
-from tests.app.db import create_reply_to_email, create_service_sms_sender
+from tests.app.db import create_reply_to_email, create_service_sms_sender, create_letter_contact
 
 
 # all of these tests should have redis enabled (except where we specifically disable it)
@@ -402,3 +405,49 @@ def test_check_service_sms_sender_id_where_sms_sender_is_not_found(sample_servic
     assert e.value.status_code == 400
     assert e.value.message == 'sms_sender_id {} does not exist in database for service id {}' \
         .format(fake_uuid, sample_service.id)
+
+
+def test_check_service_letter_contact_id_where_letter_contact_id_is_none():
+    assert check_service_letter_contact_id(None, None, 'letter') is None
+
+
+def test_check_service_letter_contact_id_where_letter_contact_id_is_found(sample_service):
+    letter_contact = create_letter_contact(service=sample_service, contact_block='123456')
+    assert check_service_letter_contact_id(sample_service.id, letter_contact.id, LETTER_TYPE) == '123456'
+
+
+def test_check_service_letter_contact_id_where_service_id_is_not_found(sample_service, fake_uuid):
+    letter_contact = create_letter_contact(service=sample_service, contact_block='123456')
+    with pytest.raises(BadRequestError) as e:
+        check_service_letter_contact_id(fake_uuid, letter_contact.id, LETTER_TYPE)
+    assert e.value.status_code == 400
+    assert e.value.message == 'letter_contact_id {} does not exist in database for service id {}' \
+        .format(letter_contact.id, fake_uuid)
+
+
+def test_check_service_letter_contact_id_where_letter_contact_is_not_found(sample_service, fake_uuid):
+    with pytest.raises(BadRequestError) as e:
+        check_service_letter_contact_id(sample_service.id, fake_uuid, LETTER_TYPE)
+    assert e.value.status_code == 400
+    assert e.value.message == 'letter_contact_id {} does not exist in database for service id {}' \
+        .format(fake_uuid, sample_service.id)
+
+
+@pytest.mark.parametrize('notification_type', ['sms', 'email', 'letter'])
+def test_check_reply_to_with_empty_reply_to(sample_service, notification_type):
+    assert check_reply_to(sample_service.id, None, notification_type) is None
+
+
+def test_check_reply_to_email_type(sample_service):
+    reply_to_address = create_reply_to_email(sample_service, "test@test.com")
+    assert check_reply_to(sample_service.id, reply_to_address.id, EMAIL_TYPE) == 'test@test.com'
+
+
+def test_check_reply_to_sms_type(sample_service):
+    sms_sender = create_service_sms_sender(service=sample_service, sms_sender='123456')
+    assert check_reply_to(sample_service.id, sms_sender.id, SMS_TYPE) == '123456'
+
+
+def test_check_reply_to_letter_type(sample_service):
+    letter_contact = create_letter_contact(service=sample_service, contact_block='123456')
+    assert check_reply_to(sample_service.id, letter_contact.id, LETTER_TYPE) == '123456'

tests/app/service/test_send_one_off_notification.py

@@ -18,6 +18,7 @@ from app.models import (
 from tests.app.db import (
     create_user,
     create_reply_to_email,
+    create_letter_contact,
     create_service,
     create_template
 )
@@ -217,7 +218,27 @@ def test_send_one_off_notification_should_add_email_reply_to_text_for_notificati
         research_mode=False,
         queue=None
     )
-    notification.reply_to_text == reply_to_email.email_address
+    assert notification.reply_to_text == reply_to_email.email_address
+
+
+def test_send_one_off_letter_notification_should_use_template_reply_to_text(sample_letter_template, celery_mock):
+    letter_contact = create_letter_contact(sample_letter_template.service, "Edinburgh, ED1 1AA", is_default=False)
+    sample_letter_template.reply_to = str(letter_contact.id)
+
+    data = {
+        'to': 'user@example.com',
+        'template_id': str(sample_letter_template.id),
+        'created_by': str(sample_letter_template.service.created_by_id)
+    }
+
+    notification_id = send_one_off_notification(service_id=sample_letter_template.service.id, post_data=data)
+    notification = Notification.query.get(notification_id['id'])
+    celery_mock.assert_called_once_with(
+        notification=notification,
+        research_mode=False,
+        queue=None
+    )
+    assert notification.reply_to_text == "Edinburgh, ED1 1AA"
 
 
 def test_send_one_off_notification_should_throw_exception_if_reply_to_id_doesnot_exist(

tests/app/template/test_rest.py

@@ -588,6 +588,28 @@ def test_create_a_template_with_reply_to(admin_request, sample_user):
     assert sorted(json_resp['data']) == sorted(template_schema.dump(template).data)
 
 
+def test_create_a_template_with_foreign_service_reply_to(admin_request, sample_user):
+    service = create_service(service_permissions=['letter'])
+    service2 = create_service(service_name='test service', email_from='test@example.com',
+                              service_permissions=['letter'])
+    letter_contact = create_letter_contact(service2, "Edinburgh, ED1 1AA")
+    data = {
+        'name': 'my template',
+        'subject': 'subject',
+        'template_type': 'letter',
+        'content': 'template <b>content</b>',
+        'service': str(service.id),
+        'created_by': str(sample_user.id),
+        'reply_to': str(letter_contact.id),
+    }
+
+    json_resp = admin_request.post('template.create_template', service_id=service.id, _data=data, _expected_status=400)
+
+    assert json_resp['message'] == "letter_contact_id {} does not exist in database for service id {}".format(
+        str(letter_contact.id), str(service.id)
+    )
+
+
 def test_get_template_reply_to(client, sample_letter_template):
     auth_header = create_authorization_header()
     letter_contact = create_letter_contact(sample_letter_template.service, "Edinburgh, ED1 1AA")
@@ -621,6 +643,29 @@ def test_update_template_reply_to(client, sample_letter_template):
     assert template.reply_to == letter_contact.id
 
 
+def test_update_template_with_foreign_service_reply_to(client, sample_letter_template):
+    auth_header = create_authorization_header()
+
+    service2 = create_service(service_name='test service', email_from='test@example.com',
+                              service_permissions=['letter'])
+    letter_contact = create_letter_contact(service2, "Edinburgh, ED1 1AA")
+
+    data = {
+        'reply_to': str(letter_contact.id),
+    }
+
+    resp = client.post('/service/{}/template/{}'.format(sample_letter_template.service_id, sample_letter_template.id),
+                       data=json.dumps(data),
+                       headers=[('Content-Type', 'application/json'), auth_header])
+
+    assert resp.status_code == 400, resp.get_data(as_text=True)
+    json_resp = json.loads(resp.get_data(as_text=True))
+
+    assert json_resp['message'] == "letter_contact_id {} does not exist in database for service id {}".format(
+        str(letter_contact.id), str(sample_letter_template.service_id)
+    )
+
+
 def test_update_redact_template(admin_request, sample_template):
     assert sample_template.redact_personalisation is False
 

tests/app/test_config.py

@@ -63,7 +63,7 @@ def test_cloudfoundry_config_has_different_defaults():
 def test_queue_names_all_queues_correct():
     # Need to ensure that all_queues() only returns queue names used in API
     queues = QueueNames.all_queues()
-    assert len(queues) == 12
+    assert len(queues) == 13
     assert set([
         QueueNames.PRIORITY,
         QueueNames.PERIODIC,
@@ -76,5 +76,6 @@ def test_queue_names_all_queues_correct():
         QueueNames.RETRY,
         QueueNames.NOTIFY,
         QueueNames.CREATE_LETTERS_PDF,
-        QueueNames.CALLBACKS
+        QueueNames.CALLBACKS,
+        QueueNames.LETTERS,
     ]) == set(queues)

tests/app/v2/notifications/test_post_notifications.py

@@ -28,6 +28,7 @@ from tests.app.db import (
     create_service,
     create_template,
     create_reply_to_email,
+    create_letter_contact,
     create_service_sms_sender,
     create_service_with_inbound_number
 )
@@ -639,3 +640,26 @@ def test_post_email_notification_with_invalid_reply_to_id_returns_400(client, sa
     assert 'email_reply_to_id {} does not exist in database for service id {}'. \
         format(fake_uuid, sample_email_template.service_id) in resp_json['errors'][0]['message']
     assert 'BadRequestError' in resp_json['errors'][0]['error']
+
+
+def test_letter_notification_should_use_template_reply_to(client, sample_letter_template):
+    letter_contact = create_letter_contact(sample_letter_template.service, "Edinburgh, ED1 1AA", is_default=False)
+    sample_letter_template.reply_to = str(letter_contact.id)
+
+    data = {
+        'template_id': str(sample_letter_template.id),
+        'personalisation': {
+            'address_line_1': '123',
+            'address_line_2': '234',
+            'postcode': 'W1A1AA',
+        }
+    }
+    response = client.post("v2/notifications/letter",
+                           data=json.dumps(data),
+                           headers=[('Content-Type', 'application/json'),
+                                    create_authorization_header(service_id=sample_letter_template.service.id)]
+                           )
+    assert response.status_code == 201, response.get_data(as_text=True)
+    notifications = Notification.query.all()
+    assert len(notifications) == 1
+    assert notifications[0].reply_to_text == "Edinburgh, ED1 1AA"