Add a task to save-api-sms for high volume services.

When we initially added a new task to persist the notifications for a high volume service we wanted to implement it as quickly as possible, so ignored SMS.
This will allow a high volume service to send SMS, the SMS will be sent to a queue to then persist and send the SMS, similar to emails.

At this point I haven't added a new application to consume the new save-api-sms-tasks. But we can add a separate application or be happy with how the app scales for both email and sms.

tests/app/celery/test_tasks.py

@@ -33,8 +33,9 @@ from app.celery.tasks import (
     s3,
     send_inbound_sms_to_service,
     process_returned_letters_list,
-    save_api_email,
     get_recipient_csv_and_template_and_sender_id,
+    save_api_email,
+    save_api_sms
 )
 from app.config import QueueNames
 from app.dao import jobs_dao, service_email_reply_to_dao, service_sms_sender_dao
@@ -1791,51 +1792,66 @@ def test_process_returned_letters_populates_returned_letters_table(
 
 
 @freeze_time('2020-03-25 14:30')
-def test_save_api_email(sample_email_template, mocker):
-    mock_send_email_to_provider = mocker.patch('app.celery.provider_tasks.deliver_email.apply_async')
-    api_key = create_api_key(service=sample_email_template.service)
+@pytest.mark.parametrize('notification_type', ['sms', 'email'])
+def test_save_api_email_or_sms(mocker, sample_service, notification_type):
+    template = create_template(sample_service) if notification_type == SMS_TYPE \
+        else create_template(sample_service, template_type=EMAIL_TYPE)
+    mock_provider_task = mocker.patch(f'app.celery.provider_tasks.deliver_{notification_type}.apply_async')
+    api_key = create_api_key(service=template.service)
     data = {
         "id": str(uuid.uuid4()),
-        "template_id": str(sample_email_template.id),
-        "template_version": sample_email_template.version,
-        "to": "jane.citizen@example.com",
-        "service_id": str(sample_email_template.service_id),
+        "template_id": str(template.id),
+        "template_version": template.version,
+        "service_id": str(template.service_id),
         "personalisation": None,
-        "notification_type": sample_email_template.template_type,
+        "notification_type": template.template_type,
         "api_key_id": str(api_key.id),
         "key_type": api_key.key_type,
         "client_reference": 'our email',
-        "reply_to_text": "our.email@gov.uk",
+        "reply_to_text": None,
         "document_download_count": 0,
         "status": NOTIFICATION_CREATED,
         "created_at": datetime.utcnow().strftime(DATETIME_FORMAT),
     }
 
+    if notification_type == EMAIL_TYPE:
+        data.update({"to": "jane.citizen@example.com"})
+        expected_queue = QueueNames.SEND_EMAIL
+    else:
+        data.update({"to": "+447700900855"})
+        expected_queue = QueueNames.SEND_SMS
+
     encrypted = encryption.encrypt(
         data
     )
 
     assert len(Notification.query.all()) == 0
-    save_api_email(encrypted)
+    if notification_type == EMAIL_TYPE:
+        save_api_email(encrypted_notification=encrypted)
+    else:
+        save_api_sms(encrypted_notification=encrypted)
     notifications = Notification.query.all()
     assert len(notifications) == 1
     assert str(notifications[0].id) == data['id']
     assert notifications[0].created_at == datetime(2020, 3, 25, 14, 30)
-    mock_send_email_to_provider.assert_called_once_with([data['id']], queue=QueueNames.SEND_EMAIL)
+    assert notifications[0].notification_type == notification_type
+    mock_provider_task.assert_called_once_with([data['id']], queue=expected_queue)
 
 
 @freeze_time('2020-03-25 14:30')
-def test_save_api_email_dont_retry_if_notification_already_exists(sample_email_template, mocker):
-    mock_send_email_to_provider = mocker.patch('app.celery.provider_tasks.deliver_email.apply_async')
-    api_key = create_api_key(service=sample_email_template.service)
+@pytest.mark.parametrize('notification_type', ['sms', 'email'])
+def test_save_api_email_dont_retry_if_notification_already_exists(sample_service, mocker, notification_type):
+    template = create_template(sample_service) if notification_type == SMS_TYPE \
+        else create_template(sample_service, template_type=EMAIL_TYPE)
+    mock_provider_task = mocker.patch(f'app.celery.provider_tasks.deliver_{notification_type}.apply_async')
+    api_key = create_api_key(service=template.service)
     data = {
         "id": str(uuid.uuid4()),
-        "template_id": str(sample_email_template.id),
-        "template_version": sample_email_template.version,
-        "to": "jane.citizen@example.com",
-        "service_id": str(sample_email_template.service_id),
+        "template_id": str(template.id),
+        "template_version": template.version,
+        "service_id": str(template.service_id),
         "personalisation": None,
-        "notification_type": sample_email_template.template_type,
+        "notification_type": template.template_type,
         "api_key_id": str(api_key.id),
         "key_type": api_key.key_type,
         "client_reference": 'our email',
@@ -1845,18 +1861,32 @@ def test_save_api_email_dont_retry_if_notification_already_exists(sample_email_t
         "created_at": datetime.utcnow().strftime(DATETIME_FORMAT),
     }
 
+    if notification_type == EMAIL_TYPE:
+        data.update({"to": "jane.citizen@example.com"})
+        expected_queue = QueueNames.SEND_EMAIL
+    else:
+        data.update({"to": "+447700900855"})
+        expected_queue = QueueNames.SEND_SMS
+
     encrypted = encryption.encrypt(
         data
     )
     assert len(Notification.query.all()) == 0
-    save_api_email(encrypted)
+
+    if notification_type == EMAIL_TYPE:
+        save_api_email(encrypted_notification=encrypted)
+    else:
+        save_api_sms(encrypted_notification=encrypted)
     notifications = Notification.query.all()
     assert len(notifications) == 1
     # call the task again with the same notification
-    save_api_email(encrypted)
+    if notification_type == EMAIL_TYPE:
+        save_api_email(encrypted_notification=encrypted)
+    else:
+        save_api_sms(encrypted_notification=encrypted)
     notifications = Notification.query.all()
     assert len(notifications) == 1
     assert str(notifications[0].id) == data['id']
     assert notifications[0].created_at == datetime(2020, 3, 25, 14, 30)
     # should only have sent the notification once.
-    mock_send_email_to_provider.assert_called_once_with([data['id']], queue=QueueNames.SEND_EMAIL)
+    mock_provider_task.assert_called_once_with([data['id']], queue=expected_queue)

tests/app/test_config.py

@@ -60,7 +60,7 @@ def test_load_config_if_cloudfoundry_not_available(reload_config):
 def test_queue_names_all_queues_correct():
     # Need to ensure that all_queues() only returns queue names used in API
     queues = QueueNames.all_queues()
-    assert len(queues) == 15
+    assert len(queues) == 16
     assert set([
         QueueNames.PRIORITY,
         QueueNames.PERIODIC,
@@ -76,5 +76,6 @@ def test_queue_names_all_queues_correct():
         QueueNames.CALLBACKS,
         QueueNames.LETTERS,
         QueueNames.SMS_CALLBACKS,
-        QueueNames.SAVE_API_EMAIL
+        QueueNames.SAVE_API_EMAIL,
+        QueueNames.SAVE_API_SMS
     ]) == set(queues)

tests/app/v2/notifications/test_post_notifications.py

@@ -1029,12 +1029,44 @@ def test_post_email_notification_when_data_is_empty_returns_400(client, sample_s
         assert error_msg == 'email_address is a required property'
 
 
+def test_post_notifications_saves_sms_to_queue(client, notify_db_session, mocker):
+    save_task = mocker.patch("app.celery.tasks.save_api_sms.apply_async")
+    mock_send_task = mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async')
+
+    service = create_service(
+        service_id=current_app.config['HIGH_VOLUME_SERVICE'][0],
+        service_name=f'high volume service',
+    )
+    template = create_template(service=service, content='((message))')
+    data = {
+        "phone_number": "+447700900855",
+        "template_id": template.id,
+        "personalisation": {"message": "Dear citizen, have a nice day"}
+    }
+
+    response = client.post(
+        path=f'/v2/notifications/sms',
+        data=json.dumps(data),
+        headers=[('Content-Type', 'application/json'), create_authorization_header(service_id=service.id)]
+    )
+
+    json_resp = response.get_json()
+
+    assert response.status_code == 201
+    assert json_resp['id']
+    assert json_resp['content']['body'] == "Dear citizen, have a nice day"
+    assert json_resp['template']['id'] == str(template.id)
+    save_task.assert_called_once_with([mock.ANY], queue='save-api-sms-tasks')
+    assert not mock_send_task.called
+    assert len(Notification.query.all()) == 0
+
+
 def test_post_notifications_saves_email_to_queue(client, notify_db_session, mocker):
     save_email_task = mocker.patch("app.celery.tasks.save_api_email.apply_async")
     mock_send_task = mocker.patch('app.celery.provider_tasks.deliver_email.apply_async')
 
     service = create_service(
-        service_id=current_app.config['HIGH_VOLUME_SERVICE'][0],
+        service_id=current_app.config['HIGH_VOLUME_SERVICE'][1],
         service_name='high volume service',
     )
     template = create_template(service=service, content='((message))', template_type=EMAIL_TYPE)
@@ -1068,7 +1100,7 @@ def test_post_notifications_saves_email_normally_if_save_email_to_queue_fails(cl
     mock_send_task = mocker.patch('app.celery.provider_tasks.deliver_email.apply_async')
 
     service = create_service(
-        service_id=current_app.config['HIGH_VOLUME_SERVICE'][1],
+        service_id=current_app.config['HIGH_VOLUME_SERVICE'][2],
         service_name='high volume service',
     )
     template = create_template(service=service, content='((message))', template_type=EMAIL_TYPE)
@@ -1100,7 +1132,7 @@ def test_post_notifications_doesnt_save_email_to_queue_for_test_emails(client, n
     mock_send_task = mocker.patch('app.celery.provider_tasks.deliver_email.apply_async')
 
     service = create_service(
-        service_id=current_app.config['HIGH_VOLUME_SERVICE'][2],
+        service_id=current_app.config['HIGH_VOLUME_SERVICE'][3],
         service_name='high volume service',
     )
     template = create_template(service=service, content='((message))', template_type=EMAIL_TYPE)
@@ -1125,33 +1157,3 @@ def test_post_notifications_doesnt_save_email_to_queue_for_test_emails(client, n
     assert mock_send_task.called
     assert not save_email_task.called
     assert len(Notification.query.all()) == 1
-
-
-def test_post_notifications_doesnt_save_email_to_queue_for_sms(client, notify_db_session, mocker):
-    save_email_task = mocker.patch("app.celery.tasks.save_api_email.apply_async")
-    mock_send_task = mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async')
-
-    service = create_service(
-        service_id=current_app.config['HIGH_VOLUME_SERVICE'][3],
-        service_name='high volume service',
-    )
-    template = create_template(service=service, content='((message))', template_type=SMS_TYPE)
-    data = {
-        "phone_number": '+447700900855',
-        "template_id": template.id,
-        "personalisation": {"message": "Dear citizen, have a nice day"}
-    }
-    response = client.post(
-        path='/v2/notifications/sms',
-        data=json.dumps(data),
-        headers=[('Content-Type', 'application/json'), create_authorization_header(service_id=service.id)]
-    )
-
-    json_resp = response.get_json()
-
-    assert response.status_code == 201
-    assert json_resp['id']
-    assert mock_send_task.called
-    assert not save_email_task.called
-
-    assert len(Notification.query.all()) == 1