Merge pull request #866 from alphagov/no-downtime-hosting-migration

Reinstating the 2 task model for API submitted notifications.

app/celery/tasks.py

@@ -123,6 +123,39 @@ def process_row(row_number, recipient, personalisation, template, job, service):
     )
 
 
+def send_notification_to_persist_queue(
+        notification_id, service, template_type, encrypted, priority=False, research_mode=False
+):
+    queues = {
+        SMS_TYPE: 'db-sms',
+        EMAIL_TYPE: 'db-email'
+    }
+
+    send_fns = {
+        SMS_TYPE: send_sms,
+        EMAIL_TYPE: send_email
+    }
+
+    send_fn = send_fns[template_type]
+
+    if research_mode:
+        queue_name = "research-mode"
+    elif priority:
+        queue_name = "notify"
+    else:
+        queue_name = queues[template_type]
+
+    send_fn.apply_async(
+        (
+            str(service.id),
+            notification_id,
+            encrypted,
+            datetime.utcnow().strftime(DATETIME_FORMAT)
+        ),
+        queue=queue_name
+    )
+
+
 def __sending_limits_for_job_exceeded(service, job, job_id):
     total_sent = fetch_todays_total_message_count(service.id)
 

app/notifications/process_notifications.py

@@ -5,6 +5,7 @@ from flask import current_app
 from app import redis_store
 from app.celery import provider_tasks
 from notifications_utils.clients import redis
+
 from app.dao.notifications_dao import dao_create_notification, dao_delete_notifications_and_history_by_id
 from app.models import SMS_TYPE, Notification, KEY_TYPE_TEST, EMAIL_TYPE
 from app.v2.errors import BadRequestError, SendNotificationToQueueError
@@ -37,7 +38,9 @@ def persist_notification(template_id,
                          job_row_number=None,
                          reference=None,
                          notification_id=None,
-                         simulated=False):
+                         simulated=False,
+                         persist=True):
+
     # if simulated create a Notification model to return but do not persist the Notification to the dB
     notification = Notification(
         id=notification_id,
@@ -56,7 +59,8 @@ def persist_notification(template_id,
         client_reference=reference
     )
     if not simulated:
-        dao_create_notification(notification)
+        if persist:
+            dao_create_notification(notification)
         if redis_store.get(redis.daily_limit_cache_key(service.id)):
             redis_store.incr(redis.daily_limit_cache_key(service.id))
         if redis_store.get_all_from_hash(cache_key_for_service_template_counter(service.id)):

app/notifications/rest.py

@@ -6,13 +6,14 @@ from flask import (
     json
 )
 
-from app import api_user
+from app import api_user, encryption, create_uuid
+from app.celery import tasks
 from app.dao import (
     templates_dao,
     services_dao,
     notifications_dao
 )
-from app.models import KEY_TYPE_TEAM, PRIORITY
+from app.models import KEY_TYPE_TEAM, PRIORITY, KEY_TYPE_TEST
 from app.models import SMS_TYPE
 from app.notifications.process_client_response import (
     validate_callback_data,
@@ -42,7 +43,6 @@ from app.errors import (
     InvalidRequest
 )
 
-
 register_errors(notifications)
 
 
@@ -95,7 +95,6 @@ def get_notification_statistics_for_day():
 
 @notifications.route('/notifications/<string:notification_type>', methods=['POST'])
 def send_notification(notification_type):
-
     if notification_type not in ['sms', 'email']:
         assert False
 
@@ -121,20 +120,44 @@ def send_notification(notification_type):
 
     # Do not persist or send notification to the queue if it is a simulated recipient
     simulated = simulated_recipient(notification_form['to'], notification_type)
-    notification_model = persist_notification(template_id=template.id,
+    notification_model = persist_notification(
-                                              template_version=template.version,
+        notification_id=create_uuid(),
-                                              recipient=notification_form['to'],
+        template_id=template.id,
-                                              service=service,
+        template_version=template.version,
-                                              personalisation=notification_form.get('personalisation', None),
+        recipient=notification_form['to'],
-                                              notification_type=notification_type,
+        service=service,
-                                              api_key_id=api_user.id,
+        personalisation=notification_form.get('personalisation', None),
-                                              key_type=api_user.key_type,
+        notification_type=notification_type,
-                                              simulated=simulated)
+        api_key_id=api_user.id,
+        key_type=api_user.key_type,
+        simulated=simulated,
+        persist=False)
+
+    notification_data = {
+        'template': str(template.id),
+        'template_version': template.version,
+        'to': notification_form['to']
+    }
+
+    if notification_model.personalisation:
+        notification_data.update({
+            'personalisation': dict(notification_model.personalisation)
+        })
+    encrypted = encryption.encrypt(notification_data)
+
     if not simulated:
-        queue_name = 'notify' if template.process_type == PRIORITY else None
+        tasks.send_notification_to_persist_queue(
-        send_notification_to_queue(notification=notification_model,
+            notification_model.id,
-                                   research_mode=service.research_mode,
+            service,
-                                   queue=queue_name)
+            template.template_type,
+            encrypted,
+            template.process_type == PRIORITY,
+            service.research_mode or api_user.key_type == KEY_TYPE_TEST
+        )
+        # queue_name = 'notify' if template.process_type == PRIORITY else None
+        # send_notification_to_queue(notification=notification_model,
+        #                            research_mode=service.research_mode,
+        #                            queue=queue_name)
     else:
         current_app.logger.info("POST simulated notification for id: {}".format(notification_model.id))
     notification_form.update({"template_version": template.version})

app/v2/notifications/post_notifications.py

@@ -1,9 +1,10 @@
 from flask import request, jsonify, current_app
 from sqlalchemy.orm.exc import NoResultFound
 
-from app import api_user
+from app import api_user, encryption, create_uuid
+from app.celery import tasks
 from app.dao import services_dao, templates_dao
-from app.models import SMS_TYPE, EMAIL_TYPE, PRIORITY
+from app.models import SMS_TYPE, EMAIL_TYPE, PRIORITY, KEY_TYPE_TEST
 from app.notifications.process_notifications import (create_content_for_notification,
                                                      persist_notification,
                                                      send_notification_to_queue,
@@ -39,19 +40,44 @@ def post_notification(notification_type):
 
     # Do not persist or send notification to the queue if it is a simulated recipient
     simulated = simulated_recipient(send_to, notification_type)
-    notification = persist_notification(template_id=template.id,
+    notification = persist_notification(
-                                        template_version=template.version,
+        notification_id=create_uuid(),
-                                        recipient=send_to,
+        template_id=template.id,
-                                        service=service,
+        template_version=template.version,
-                                        personalisation=form.get('personalisation', None),
+        recipient=send_to,
-                                        notification_type=notification_type,
+        service=service,
-                                        api_key_id=api_user.id,
+        personalisation=form.get('personalisation', {}),
-                                        key_type=api_user.key_type,
+        notification_type=notification_type,
-                                        reference=form.get('reference', None),
+        api_key_id=api_user.id,
-                                        simulated=simulated)
+        key_type=api_user.key_type,
+        reference=form.get('reference', None),
+        simulated=simulated,
+        persist=False)
+
+    notification_data = {
+        'template': str(template.id),
+        'template_version': template.version,
+        'to': send_to
+    }
+    if notification.personalisation:
+        notification_data.update({
+            'personalisation': dict(notification.personalisation)
+        })
+
+    encrypted = encryption.encrypt(notification_data)
+
     if not simulated:
-        queue_name = 'notify' if template.process_type == PRIORITY else None
+        tasks.send_notification_to_persist_queue(
-        send_notification_to_queue(notification=notification, research_mode=service.research_mode, queue=queue_name)
+            notification.id,
+            service,
+            template.template_type,
+            encrypted,
+            template.process_type == PRIORITY,
+            service.research_mode or api_user.key_type == KEY_TYPE_TEST
+        )
+        # not doing this during paas migration
+        # queue_name = 'notify' if template.process_type == PRIORITY else None
+        # send_notification_to_queue(notification=notification, research_mode=service.research_mode, queue=queue_name)
     else:
         current_app.logger.info("POST simulated notification for id: {}".format(notification.id))
     if notification_type == SMS_TYPE:

tests/app/celery/test_tasks.py

@@ -12,7 +12,7 @@ from celery.exceptions import Retry
 from app import (encryption, DATETIME_FORMAT)
 from app.celery import provider_tasks
 from app.celery import tasks
-from app.celery.tasks import s3, build_dvla_file
+from app.celery.tasks import s3, build_dvla_file, send_notification_to_persist_queue
 from app.celery.tasks import (
     process_job,
     process_row,
@@ -66,6 +66,84 @@ def email_job_with_placeholders(notify_db, notify_db_session, sample_email_templ
     return sample_job(notify_db, notify_db_session, template=sample_email_template_with_placeholders)
 
 
+@freeze_time("2016-01-01 11:09:00.061258")
+def test_should_put_sms_messages_in_db_sms_queue(sample_service, sample_template, mocker):
+    mocker.patch('app.celery.tasks.send_sms.apply_async')
+    send_notification_to_persist_queue("notification-id", sample_service, sample_template.template_type, "encrypted")
+
+    tasks.send_sms.apply_async.assert_called_once_with(
+        (str(sample_service.id),
+         "notification-id",
+         "encrypted",
+         "2016-01-01T11:09:00.061258Z"),
+        queue="db-sms"
+    )
+
+
+@freeze_time("2016-01-01 11:09:00.061258")
+def test_should_put_messages_in_priority_queue(sample_service, sample_template, mocker):
+    mocker.patch('app.celery.tasks.send_sms.apply_async')
+    send_notification_to_persist_queue(
+        "notification-id", sample_service, sample_template.template_type, "encrypted", True
+    )
+
+    tasks.send_sms.apply_async.assert_called_once_with(
+        (str(sample_service.id),
+         "notification-id",
+         "encrypted",
+         "2016-01-01T11:09:00.061258Z"),
+        queue="notify"
+    )
+
+
+@freeze_time("2016-01-01 11:09:00.061258")
+def test_should_put_messages_in_research_mode_queue(sample_service, sample_template, mocker):
+    mocker.patch('app.celery.tasks.send_sms.apply_async')
+    send_notification_to_persist_queue(
+        "notification-id", sample_service, sample_template.template_type, "encrypted", False, True
+    )
+
+    tasks.send_sms.apply_async.assert_called_once_with(
+        (str(sample_service.id),
+         "notification-id",
+         "encrypted",
+         "2016-01-01T11:09:00.061258Z"),
+        queue="research-mode"
+    )
+
+
+@freeze_time("2016-01-01 11:09:00.061258")
+def test_should_put_messages_in_research_mode_queue_overriding_priority_mode(sample_service, sample_template, mocker):
+    mocker.patch('app.celery.tasks.send_sms.apply_async')
+    send_notification_to_persist_queue(
+        "notification-id", sample_service, sample_template.template_type, "encrypted", True, True
+    )
+
+    tasks.send_sms.apply_async.assert_called_once_with(
+        (str(sample_service.id),
+         "notification-id",
+         "encrypted",
+         "2016-01-01T11:09:00.061258Z"),
+        queue="research-mode"
+    )
+
+
+@freeze_time("2016-01-01 11:09:00.061258")
+def test_should_put_email_messages_in_db_email_queue(sample_service, sample_email_template, mocker):
+    mocker.patch('app.celery.tasks.send_email.apply_async')
+    send_notification_to_persist_queue(
+        "notification-id", sample_service, sample_email_template.template_type, "encrypted"
+    )
+
+    tasks.send_email.apply_async.assert_called_once_with(
+        (str(sample_service.id),
+         "notification-id",
+         "encrypted",
+         "2016-01-01T11:09:00.061258Z"),
+        queue="db-email"
+    )
+
+
 # -------------- process_job tests -------------- #
 
 

tests/app/notifications/rest/test_send_notification.py

@@ -104,7 +104,9 @@ def test_send_notification_invalid_template_id(notify_api, sample_template, mock
 def test_send_notification_with_placeholders_replaced(notify_api, sample_email_template_with_placeholders, mocker):
     with notify_api.test_request_context():
         with notify_api.test_client() as client:
-            mocked = mocker.patch('app.celery.provider_tasks.deliver_email.apply_async')
+            mocked = mocker.patch('app.celery.tasks.send_email.apply_async')
+            mocker.patch('app.encryption.encrypt', return_value="something_encrypted")
+            mocker.patch('app.encryption.decrypt', return_value={"key": "something_encrypted"})
 
             data = {
                 'to': 'ok@ok.com',
@@ -125,8 +127,13 @@ def test_send_notification_with_placeholders_replaced(notify_api, sample_email_t
             data.update({"template_version": sample_email_template_with_placeholders.version})
 
             mocked.assert_called_once_with(
-                [notification_id],
+                (
-                queue="send-email"
+                    str(sample_email_template_with_placeholders.service.id),
+                    notification_id,
+                    "something_encrypted",
+                    '2016-01-01T11:09:00.061258Z'
+                ),
+                queue="db-email"
             )
             assert response.status_code == 201
             assert response_data['body'] == u'Hello Jo\nThis is an email from GOV.\u200BUK'
@@ -199,7 +206,7 @@ def test_should_send_notification_if_restricted_and_a_service_user(notify_api,
                                                                    mocker):
     with notify_api.test_request_context():
         with notify_api.test_client() as client:
-            mocked = mocker.patch('app.celery.provider_tasks.deliver_{}.apply_async'.format(template_type))
+            mocked = mocker.patch('app.celery.tasks.send_{}.apply_async'.format(template_type))
 
             template = sample_template if template_type == 'sms' else sample_email_template
             to = template.service.created_by.mobile_number if template_type == 'sms' \
@@ -260,7 +267,7 @@ def test_should_not_allow_template_from_another_service(notify_api,
 def test_should_allow_valid_sms_notification(notify_api, sample_template, mocker):
     with notify_api.test_request_context():
         with notify_api.test_client() as client:
-            mocked = mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async')
+            mocked = mocker.patch('app.celery.tasks.send_sms.apply_async')
             mocker.patch('app.encryption.encrypt', return_value="something_encrypted")
 
             data = {
@@ -278,7 +285,12 @@ def test_should_allow_valid_sms_notification(notify_api, sample_template, mocker
             response_data = json.loads(response.data)['data']
             notification_id = response_data['notification']['id']
 
-            mocked.assert_called_once_with([notification_id], queue='send-sms')
+            mocked.assert_called_once_with(
+                (str(sample_template.service.id),
+                 notification_id,
+                 "something_encrypted",
+                 '2016-01-01T11:09:00.061258Z'),
+                queue='db-sms')
             assert response.status_code == 201
             assert notification_id
             assert 'subject' not in response_data
@@ -309,11 +321,11 @@ def test_should_reject_email_notification_with_bad_email(notify_api, sample_emai
             assert data['message']['to'][0] == 'Not a valid email address'
 
 
-@freeze_time("2016-01-01 11:09:00.061258")
+@freeze_time("2016-01-01 11:00:00.000000")
 def test_should_allow_valid_email_notification(notify_api, sample_email_template, mocker):
     with notify_api.test_request_context():
         with notify_api.test_client() as client:
-            mocker.patch('app.celery.provider_tasks.deliver_email.apply_async')
+            mocker.patch('app.celery.tasks.send_email.apply_async')
             mocker.patch('app.encryption.encrypt', return_value="something_encrypted")
 
             data = {
@@ -330,9 +342,15 @@ def test_should_allow_valid_email_notification(notify_api, sample_email_template
             assert response.status_code == 201
             response_data = json.loads(response.get_data(as_text=True))['data']
             notification_id = response_data['notification']['id']
-            app.celery.provider_tasks.deliver_email.apply_async.assert_called_once_with(
+            app.celery.tasks.send_email.apply_async.assert_called_once_with(
-                [notification_id],
+                (
-                queue="send-email"
+                    str(sample_email_template.service.id),
+                    notification_id,
+                    "something_encrypted",
+                    '2016-01-01T11:00:00.000000Z'
+
+                ),
+                queue="db-email"
             )
 
             assert response.status_code == 201
@@ -418,7 +436,7 @@ def test_should_allow_api_call_if_under_day_limit_regardless_of_type(
         restricted):
     with notify_api.test_request_context():
         with notify_api.test_client() as client:
-            mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async')
+            mocker.patch('app.celery.tasks.send_sms.apply_async')
             mocker.patch('app.encryption.encrypt', return_value="something_encrypted")
 
             service = create_sample_service(notify_db, notify_db_session, limit=2, restricted=restricted)
@@ -444,7 +462,7 @@ def test_should_allow_api_call_if_under_day_limit_regardless_of_type(
 def test_should_not_return_html_in_body(notify_api, notify_db, notify_db_session, mocker):
     with notify_api.test_request_context():
         with notify_api.test_client() as client:
-            mocker.patch('app.celery.provider_tasks.deliver_email.apply_async')
+            mocker.patch('app.celery.tasks.send_email.apply_async')
             email_template = create_sample_email_template(notify_db, notify_db_session, content='hello\nthere')
 
             data = {
@@ -464,7 +482,7 @@ def test_should_not_return_html_in_body(notify_api, notify_db, notify_db_session
 
 def test_should_not_send_email_if_team_api_key_and_not_a_service_user(notify_api, sample_email_template, mocker):
     with notify_api.test_request_context(), notify_api.test_client() as client:
-        mocker.patch('app.celery.provider_tasks.deliver_email.apply_async')
+        mocker.patch('app.celery.tasks.send_email.apply_async')
         data = {
             'to': "not-someone-we-trust@email-address.com",
             'template': str(sample_email_template.id),
@@ -479,17 +497,15 @@ def test_should_not_send_email_if_team_api_key_and_not_a_service_user(notify_api
 
         json_resp = json.loads(response.get_data(as_text=True))
 
-        app.celery.provider_tasks.deliver_email.apply_async.assert_not_called()
+        app.celery.tasks.send_email.apply_async.assert_not_called()
 
         assert response.status_code == 400
-        assert [
+        assert ['Can’t send to this recipient using a team-only API key'] == json_resp['message']['to']
-            'Can’t send to this recipient using a team-only API key'
-        ] == json_resp['message']['to']
 
 
 def test_should_not_send_sms_if_team_api_key_and_not_a_service_user(notify_api, sample_template, mocker):
     with notify_api.test_request_context(), notify_api.test_client() as client:
-        mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async')
+        mocker.patch('app.celery.tasks.send_sms.apply_async')
 
         data = {
             'to': '07123123123',
@@ -504,18 +520,18 @@ def test_should_not_send_sms_if_team_api_key_and_not_a_service_user(notify_api,
             headers=[('Content-Type', 'application/json'), auth_header])
 
         json_resp = json.loads(response.get_data(as_text=True))
-        app.celery.provider_tasks.deliver_sms.apply_async.assert_not_called()
+        app.celery.tasks.send_sms.apply_async.assert_not_called()
 
         assert response.status_code == 400
-        assert [
+        assert ['Can’t send to this recipient using a team-only API key'] == json_resp['message']['to']
-            'Can’t send to this recipient using a team-only API key'
-        ] == json_resp['message']['to']
 
 
+@freeze_time("2016-01-01 11:00:00.000000")
 def test_should_send_email_if_team_api_key_and_a_service_user(notify_api, sample_email_template, fake_uuid, mocker):
     with notify_api.test_request_context(), notify_api.test_client() as client:
-        mocker.patch('app.celery.provider_tasks.deliver_email.apply_async')
+        mocker.patch('app.celery.tasks.send_email.apply_async')
         mocker.patch('app.dao.notifications_dao.create_uuid', return_value=fake_uuid)
+        mocker.patch('app.encryption.encrypt', return_value="something_encrypted")
 
         data = {
             'to': sample_email_template.service.created_by.email_address,
@@ -532,18 +548,27 @@ def test_should_send_email_if_team_api_key_and_a_service_user(notify_api, sample
             path='/notifications/email',
             data=json.dumps(data),
             headers=[('Content-Type', 'application/json'), ('Authorization', 'Bearer {}'.format(auth_header))])
+        notification_id = json.loads(response.data)['data']['notification']['id']
 
-        app.celery.provider_tasks.deliver_email.apply_async.assert_called_once_with([fake_uuid], queue='send-email')
+        app.celery.tasks.send_email.apply_async.assert_called_once_with(
+            (str(sample_email_template.service.id),
+             notification_id,
+             "something_encrypted",
+             "2016-01-01T11:00:00.000000Z"),
+            queue='db-email'
+        )
         assert response.status_code == 201
 
 
 @pytest.mark.parametrize('restricted', [True, False])
 @pytest.mark.parametrize('limit', [0, 1])
+@freeze_time("2016-01-01 11:00:00.000000")
 def test_should_send_sms_to_anyone_with_test_key(
-    notify_api, sample_template, mocker, restricted, limit, fake_uuid
+        notify_api, sample_template, mocker, restricted, limit, fake_uuid
 ):
     with notify_api.test_request_context(), notify_api.test_client() as client:
-        mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async')
+        mocker.patch('app.encryption.encrypt', return_value="something_encrypted")
+        mocker.patch('app.celery.tasks.send_sms.apply_async')
         mocker.patch('app.dao.notifications_dao.create_uuid', return_value=fake_uuid)
 
         data = {
@@ -566,18 +591,23 @@ def test_should_send_sms_to_anyone_with_test_key(
             data=json.dumps(data),
             headers=[('Content-Type', 'application/json'), ('Authorization', 'Bearer {}'.format(auth_header))]
         )
-        app.celery.provider_tasks.deliver_sms.apply_async.assert_called_once_with([fake_uuid], queue='research-mode')
+        notification_id = json.loads(response.data)['data']['notification']['id']
+        app.celery.tasks.send_sms.apply_async.assert_called_once_with(
+            (str(sample_template.service.id), notification_id, "something_encrypted", "2016-01-01T11:00:00.000000Z"),
+            queue='research-mode'
+        )
         assert response.status_code == 201
 
 
 @pytest.mark.parametrize('restricted', [True, False])
 @pytest.mark.parametrize('limit', [0, 1])
+@freeze_time("2016-01-01 11:00:00.000000")
 def test_should_send_email_to_anyone_with_test_key(
-    notify_api, sample_email_template, mocker, restricted, limit, fake_uuid
+        notify_api, sample_email_template, mocker, restricted, limit, fake_uuid
 ):
     with notify_api.test_request_context(), notify_api.test_client() as client:
-        mocker.patch('app.celery.provider_tasks.deliver_email.apply_async')
+        mocker.patch('app.celery.tasks.send_email.apply_async')
-        mocker.patch('app.dao.notifications_dao.create_uuid', return_value=fake_uuid)
+        mocker.patch('app.encryption.encrypt', return_value="something_encrypted")
 
         data = {
             'to': 'anyone123@example.com',
@@ -599,15 +629,23 @@ def test_should_send_email_to_anyone_with_test_key(
             data=json.dumps(data),
             headers=[('Content-Type', 'application/json'), ('Authorization', 'Bearer {}'.format(auth_header))]
         )
-
+        notification_id = json.loads(response.data)['data']['notification']['id']
-        app.celery.provider_tasks.deliver_email.apply_async.assert_called_once_with([fake_uuid], queue='research-mode')
+        app.celery.tasks.send_email.apply_async.assert_called_once_with(
+            (str(sample_email_template.service.id),
+             notification_id,
+             "something_encrypted",
+             "2016-01-01T11:00:00.000000Z"),
+            queue='research-mode'
+        )
         assert response.status_code == 201
 
 
+@freeze_time("2016-01-01 11:00:00.000000")
 def test_should_send_sms_if_team_api_key_and_a_service_user(notify_api, sample_template, fake_uuid, mocker):
     with notify_api.test_request_context(), notify_api.test_client() as client:
-        mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async')
+        mocker.patch('app.celery.tasks.send_sms.apply_async')
         mocker.patch('app.dao.notifications_dao.create_uuid', return_value=fake_uuid)
+        mocker.patch('app.encryption.encrypt', return_value="something_encrypted")
 
         data = {
             'to': sample_template.service.created_by.mobile_number,
@@ -625,18 +663,28 @@ def test_should_send_sms_if_team_api_key_and_a_service_user(notify_api, sample_t
             data=json.dumps(data),
             headers=[('Content-Type', 'application/json'), ('Authorization', 'Bearer {}'.format(auth_header))])
 
-        app.celery.provider_tasks.deliver_sms.apply_async.assert_called_once_with([fake_uuid], queue='send-sms')
+        notification_id = json.loads(response.data)['data']['notification']['id']
+        app.celery.tasks.send_sms.apply_async.assert_called_once_with(
+            (str(sample_template.service.id),
+             notification_id,
+             "something_encrypted",
+             "2016-01-01T11:00:00.000000Z"),
+            queue='db-sms'
+        )
         assert response.status_code == 201
 
 
 @pytest.mark.parametrize('template_type',
                          ['sms', 'email'])
+@freeze_time("2016-01-01 11:00:00.000000")
 def test_should_persist_notification(notify_api, sample_template,
                                      sample_email_template,
                                      template_type,
                                      fake_uuid, mocker):
     with notify_api.test_request_context(), notify_api.test_client() as client:
-        mocked = mocker.patch('app.celery.provider_tasks.deliver_{}.apply_async'.format(template_type))
+        mocker.patch('app.encryption.encrypt', return_value="something_encrypted")
+
+        mocked = mocker.patch('app.celery.tasks.send_{}.apply_async'.format(template_type))
         mocker.patch('app.dao.notifications_dao.create_uuid', return_value=fake_uuid)
         template = sample_template if template_type == 'sms' else sample_email_template
         to = sample_template.service.created_by.mobile_number if template_type == 'sms' \
@@ -658,17 +706,19 @@ def test_should_persist_notification(notify_api, sample_template,
             data=json.dumps(data),
             headers=[('Content-Type', 'application/json'), ('Authorization', 'Bearer {}'.format(auth_header))])
 
-        mocked.assert_called_once_with([fake_uuid], queue='send-{}'.format(template_type))
+        notification_id = json.loads(response.data)['data']['notification']['id']
+
+        mocked.assert_called_once_with(
+            (str(template.service.id), notification_id, "something_encrypted", '2016-01-01T11:00:00.000000Z'),
+            queue='db-{}'.format(template_type))
         assert response.status_code == 201
 
-        notification = notifications_dao.get_notification_by_id(fake_uuid)
+        assert not notifications_dao.get_notification_by_id(fake_uuid)
-        assert notification.to == to
-        assert notification.template_id == template.id
-        assert notification.notification_type == template_type
 
 
 @pytest.mark.parametrize('template_type',
                          ['sms', 'email'])
+@pytest.mark.skip(reason="This is not required for the duration of PaaS migration")
 def test_should_delete_notification_and_return_error_if_sqs_fails(
         client,
         sample_email_template,
@@ -770,8 +820,7 @@ def test_should_not_persist_notification_or_send_sms_if_simulated_number(
 ])
 @pytest.mark.parametrize('notification_type, to, _create_sample_template', [
     ('sms', '07827992635', create_sample_template),
-    ('email', 'non_whitelist_recipient@mail.com', create_sample_email_template)]
+    ('email', 'non_whitelist_recipient@mail.com', create_sample_email_template)])
-)
 def test_should_not_send_notification_to_non_whitelist_recipient_in_trial_mode(
     client,
     notify_db,
@@ -825,21 +874,20 @@ def test_should_not_send_notification_to_non_whitelist_recipient_in_trial_mode(
 ])
 @pytest.mark.parametrize('notification_type, to, _create_sample_template', [
     ('sms', '07123123123', create_sample_template),
-    ('email', 'whitelist_recipient@mail.com', create_sample_email_template)]
+    ('email', 'whitelist_recipient@mail.com', create_sample_email_template)])
-)
 def test_should_send_notification_to_whitelist_recipient(
-    client,
+        client,
-    notify_db,
+        notify_db,
-    notify_db_session,
+        notify_db_session,
-    notification_type,
+        notification_type,
-    to,
+        to,
-    _create_sample_template,
+        _create_sample_template,
-    key_type,
+        key_type,
-    service_restricted,
+        service_restricted,
-    mocker
+        mocker
 ):
     service = create_sample_service(notify_db, notify_db_session, limit=2, restricted=service_restricted)
-    apply_async = mocker.patch('app.celery.provider_tasks.deliver_{}.apply_async'.format(notification_type))
+    apply_async = mocker.patch('app.celery.tasks.send_{}.apply_async'.format(notification_type))
     template = _create_sample_template(notify_db, notify_db_session, service=service)
     if notification_type == 'sms':
         service_whitelist = create_sample_service_whitelist(notify_db, notify_db_session,
@@ -959,6 +1007,7 @@ def test_create_template_raises_invalid_request_when_content_too_large(
 @pytest.mark.parametrize("notification_type, send_to",
                          [("sms", "07700 900 855"),
                           ("email", "sample@email.com")])
+@freeze_time("2016-01-01 11:00:00.000000")
 def test_send_notification_uses_priority_queue_when_template_is_marked_as_priority(client, notify_db,
                                                                                    notify_db_session, mocker,
                                                                                    notification_type, send_to):
@@ -968,7 +1017,8 @@ def test_send_notification_uses_priority_queue_when_template_is_marked_as_priori
         template_type=notification_type,
         process_type='priority'
     )
-    mocked = mocker.patch('app.celery.provider_tasks.deliver_{}.apply_async'.format(notification_type))
+    mocker.patch('app.encryption.encrypt', return_value="something_encrypted")
+    mocked = mocker.patch('app.celery.tasks.send_{}.apply_async'.format(notification_type))
 
     data = {
         'to': send_to,
@@ -986,4 +1036,6 @@ def test_send_notification_uses_priority_queue_when_template_is_marked_as_priori
     notification_id = response_data['notification']['id']
 
     assert response.status_code == 201
-    mocked.assert_called_once_with([notification_id], queue='notify')
+    mocked.assert_called_once_with(
+        (str(sample.service_id), notification_id, "something_encrypted", '2016-01-01T11:00:00.000000Z'), queue='notify'
+    )

tests/app/notifications/test_process_notification.py

@@ -77,6 +77,34 @@ def test_persist_notification_creates_and_save_to_db(sample_template, sample_api
     mocked_redis.assert_called_once_with(str(sample_template.service_id) + "-2016-01-01-count")
 
 
+@freeze_time("2016-01-01 11:09:00.061258")
+def test_persist_notification_does_not_create_and_save_to_db_if_persist_is_false(
+        sample_template, sample_api_key, sample_job, mocker
+):
+    mocked_redis = mocker.patch('app.notifications.process_notifications.redis_store.get')
+
+    assert Notification.query.count() == 0
+    assert NotificationHistory.query.count() == 0
+    persist_notification(
+        sample_template.id,
+        sample_template.version,
+        '+447111111111',
+        sample_template.service,
+        {},
+        'sms',
+        sample_api_key.id,
+        sample_api_key.key_type,
+        job_id=sample_job.id,
+        job_row_number=100,
+        reference="ref",
+        persist=False)
+
+    assert Notification.query.count() == 0
+    assert NotificationHistory.query.count() == 0
+
+    mocked_redis.assert_called_once_with(str(sample_template.service_id) + "-2016-01-01-count")
+
+
 def test_persist_notification_throws_exception_when_missing_template(sample_api_key):
     assert Notification.query.count() == 0
     assert NotificationHistory.query.count() == 0

tests/app/public_contracts/test_POST_notification.py

@@ -20,7 +20,7 @@ def _post_notification(client, template, url, to):
 
 
 def test_post_sms_contract(client, mocker, sample_template):
-    mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async')
+    mocker.patch('app.celery.tasks.send_sms.apply_async')
     mocker.patch('app.encryption.encrypt', return_value="something_encrypted")
 
     response_json = return_json_from_response(_post_notification(
@@ -30,7 +30,7 @@ def test_post_sms_contract(client, mocker, sample_template):
 
 
 def test_post_email_contract(client, mocker, sample_email_template):
-    mocker.patch('app.celery.provider_tasks.deliver_email.apply_async')
+    mocker.patch('app.celery.tasks.send_email.apply_async')
     mocker.patch('app.encryption.encrypt', return_value="something_encrypted")
 
     response_json = return_json_from_response(_post_notification(

tests/app/v2/notifications/test_post_notifications.py

@@ -4,13 +4,14 @@ from flask import json
 from app.models import Notification
 from tests import create_authorization_header
 from tests.app.conftest import sample_template as create_sample_template
+from freezegun import freeze_time
 
 
 @pytest.mark.parametrize("reference", [None, "reference_from_client"])
 def test_post_sms_notification_returns_201(notify_api, sample_template_with_placeholders, mocker, reference):
     with notify_api.test_request_context():
         with notify_api.test_client() as client:
-            mocked = mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async')
+            mocked = mocker.patch('app.celery.tasks.send_sms.apply_async')
             data = {
                 'phone_number': '+447700900855',
                 'template_id': str(sample_template_with_placeholders.id),
@@ -27,14 +28,11 @@ def test_post_sms_notification_returns_201(notify_api, sample_template_with_plac
             assert response.status_code == 201
             resp_json = json.loads(response.get_data(as_text=True))
             notifications = Notification.query.all()
-            assert len(notifications) == 1
+            assert len(notifications) == 0
-            notification_id = notifications[0].id
-            assert resp_json['id'] == str(notification_id)
-            assert resp_json['reference'] == reference
             assert resp_json['content']['body'] == sample_template_with_placeholders.content.replace("(( Name))", "Jo")
             # conftest fixture service does not have a sms sender, use config default
             assert resp_json['content']['from_number'] == notify_api.config["FROM_NUMBER"]
-            assert 'v2/notifications/{}'.format(notification_id) in resp_json['uri']
+            assert 'v2/notifications/{}'.format(resp_json['id']) in resp_json['uri']
             assert resp_json['template']['id'] == str(sample_template_with_placeholders.id)
             assert resp_json['template']['version'] == sample_template_with_placeholders.version
             assert 'services/{}/templates/{}'.format(sample_template_with_placeholders.service_id,
@@ -118,7 +116,7 @@ def test_notification_returns_400_and_for_schema_problems(client, sample_templat
 
 @pytest.mark.parametrize("reference", [None, "reference_from_client"])
 def test_post_email_notification_returns_201(client, sample_email_template_with_placeholders, mocker, reference):
-    mocked = mocker.patch('app.celery.provider_tasks.deliver_email.apply_async')
+    mocked = mocker.patch('app.celery.tasks.send_email.apply_async')
     data = {
         "email_address": sample_email_template_with_placeholders.service.users[0].email_address,
         "template_id": sample_email_template_with_placeholders.id,
@@ -133,16 +131,12 @@ def test_post_email_notification_returns_201(client, sample_email_template_with_
         headers=[('Content-Type', 'application/json'), auth_header])
     assert response.status_code == 201
     resp_json = json.loads(response.get_data(as_text=True))
-    notification = Notification.query.first()
-    assert resp_json['id'] == str(notification.id)
-    assert resp_json['reference'] == reference
-    assert notification.reference is None
     assert resp_json['content']['body'] == sample_email_template_with_placeholders.content \
         .replace('((name))', 'Bob').replace('GOV.UK', u'GOV.\u200bUK')
     assert resp_json['content']['subject'] == sample_email_template_with_placeholders.subject \
         .replace('((name))', 'Bob')
     assert resp_json['content']['from_email'] == sample_email_template_with_placeholders.service.email_from
-    assert 'v2/notifications/{}'.format(notification.id) in resp_json['uri']
+    assert 'v2/notifications/{}'.format(resp_json['id']) in resp_json['uri']
     assert resp_json['template']['id'] == str(sample_email_template_with_placeholders.id)
     assert resp_json['template']['version'] == sample_email_template_with_placeholders.version
     assert 'services/{}/templates/{}'.format(str(sample_email_template_with_placeholders.service_id),
@@ -194,19 +188,21 @@ def test_should_not_persist_or_send_notification_if_simulated_recipient(
 @pytest.mark.parametrize("notification_type, key_send_to, send_to",
                          [("sms", "phone_number", "07700 900 855"),
                           ("email", "email_address", "sample@email.com")])
+@freeze_time("2016-01-01 11:00:00.000000")
 def test_send_notification_uses_priority_queue_when_template_is_marked_as_priority(client, notify_db,
                                                                                    notify_db_session,
                                                                                    mocker,
                                                                                    notification_type,
                                                                                    key_send_to,
                                                                                    send_to):
+    mocker.patch('app.encryption.encrypt', return_value="something_encrypted")
     sample = create_sample_template(
         notify_db,
         notify_db_session,
         template_type=notification_type,
         process_type='priority'
     )
-    mocked = mocker.patch('app.celery.provider_tasks.deliver_{}.apply_async'.format(notification_type))
+    mocked = mocker.patch('app.celery.tasks.send_{}.apply_async'.format(notification_type))
 
     data = {
         key_send_to: send_to,
@@ -221,6 +217,8 @@ def test_send_notification_uses_priority_queue_when_template_is_marked_as_priori
         headers=[('Content-Type', 'application/json'), auth_header])
 
     notification_id = json.loads(response.data)['id']
-
+    print(response.data)
     assert response.status_code == 201
-    mocked.assert_called_once_with([notification_id], queue='notify')
+    mocked.assert_called_once_with(
+        (str(sample.service_id), notification_id, 'something_encrypted', '2016-01-01T11:00:00.000000Z'), queue='notify'
+    )