From 052de84c9eb3982238c5d898846e8b149cab2266 Mon Sep 17 00:00:00 2001 From: Toby Lorne Date: Mon, 26 Oct 2020 17:14:08 +0000 Subject: [PATCH 1/3] clients: cbc_proxy client has canary method The CBC Proxy is essentially a lambda function which we invoke with various arguments. A way in which this can fail is that the notifications-api app invoking the function may not be able, any longer, to invoke the function. This could be caused by, for example: * an egress restriction preventing access to eu-west-2.lambda.amazonaws.com * a network partition preventing access to eu-west-2.lambda.amazonaws.com * the app's credentials have been rotated or revoked If we invoke a simple "canary" lambda function for which the app should have access to invoke, and check it for failures, we will know quickly if something is likely to be broken. This is especially important for cell broadcasts compared to email/SMS because we always have a baseline of traffic for email/SMS, and so any failure is observed almost immediately. This is not true for CB where we may expect to only see one CB message every week/month/quarter/year, as opposed to every minute or second for email/SMS. Signed-off-by: Toby Lorne Co-authored-by: Pea --- app/clients/cbc_proxy.py | 26 +++++++++ tests/app/clients/test_cbc_proxy.py | 86 +++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+) diff --git a/app/clients/cbc_proxy.py b/app/clients/cbc_proxy.py index 8d2a65d12..57296fa03 100644 --- a/app/clients/cbc_proxy.py +++ b/app/clients/cbc_proxy.py @@ -26,6 +26,12 @@ class CBCProxyNoopClient: def init_app(self, app): pass + def send_canary( + self, + identifier, + ): + pass + def create_and_send_broadcast( self, identifier, headline, description, areas @@ -57,6 +63,26 @@ class CBCProxyClient: aws_secret_access_key=app.config['CBC_PROXY_AWS_SECRET_ACCESS_KEY'], ) + def send_canary( + self, + identifier, + ): + payload_bytes = bytes(json.dumps({ + 'identifier': identifier, + }), encoding='utf8') + + result = self._lambda_client.invoke( + FunctionName='canary', + InvocationType='RequestResponse', + Payload=payload_bytes, + ) + + if result['StatusCode'] > 299: + raise Exception('Could not invoke lambda') + + if 'FunctionError' in result: + raise Exception('Function exited with unhandled exception') + def create_and_send_broadcast( self, identifier, headline, description, areas, diff --git a/tests/app/clients/test_cbc_proxy.py b/tests/app/clients/test_cbc_proxy.py index febcc26a4..8368180c7 100644 --- a/tests/app/clients/test_cbc_proxy.py +++ b/tests/app/clients/test_cbc_proxy.py @@ -1,4 +1,5 @@ import json +import uuid import pytest @@ -165,3 +166,88 @@ def test_cbc_proxy_create_and_send_handles_function_error(mocker, cbc_proxy): InvocationType='RequestResponse', Payload=mocker.ANY, ) + + +def test_cbc_proxy_send_canary_invokes_function(mocker, cbc_proxy): + identifier = str(uuid.uuid4()) + + ld_client_mock = mocker.patch.object( + cbc_proxy, + '_lambda_client', + create=True, + ) + + ld_client_mock.invoke.return_value = { + 'StatusCode': 200, + } + + cbc_proxy.send_canary( + identifier=identifier, + ) + + ld_client_mock.invoke.assert_called_once_with( + FunctionName='canary', + InvocationType='RequestResponse', + Payload=mocker.ANY, + ) + + kwargs = ld_client_mock.invoke.mock_calls[0][-1] + payload_bytes = kwargs['Payload'] + payload = json.loads(payload_bytes) + + assert payload['identifier'] == identifier + + +def test_cbc_proxy_send_canary_handles_invoke_error(mocker, cbc_proxy): + identifier = str(uuid.uuid4()) + + ld_client_mock = mocker.patch.object( + cbc_proxy, + '_lambda_client', + create=True, + ) + + ld_client_mock.invoke.return_value = { + 'StatusCode': 400, + } + + with pytest.raises(Exception) as e: + cbc_proxy.send_canary( + identifier=identifier, + ) + + assert e.match('Function exited with unhandled exception') + + ld_client_mock.invoke.assert_called_once_with( + FunctionName='canary', + InvocationType='RequestResponse', + Payload=mocker.ANY, + ) + + +def test_cbc_proxy_send_canary_handles_function_error(mocker, cbc_proxy): + identifier = str(uuid.uuid4()) + + ld_client_mock = mocker.patch.object( + cbc_proxy, + '_lambda_client', + create=True, + ) + + ld_client_mock.invoke.return_value = { + 'StatusCode': 200, + 'FunctionError': 'something', + } + + with pytest.raises(Exception) as e: + cbc_proxy.send_canary( + identifier=identifier, + ) + + assert e.match('Could not invoke lambda') + + ld_client_mock.invoke.assert_called_once_with( + FunctionName='canary', + InvocationType='RequestResponse', + Payload=mocker.ANY, + ) From be904559446c587be84cadbb06eba493b8899960 Mon Sep 17 00:00:00 2001 From: Toby Lorne Date: Mon, 26 Oct 2020 17:34:59 +0000 Subject: [PATCH 2/3] Add task to send canary to cbc proxy Create and schedule a Celery task that tests if we can send a canary message to cbc proxy. This will help us know if something happens to our connection to cbc proxy. Signed-off-by: Toby Lorne Co-authored-by: Pea Co-authored-by: Richard --- app/celery/scheduled_tasks.py | 10 +++++++++- app/config.py | 5 +++++ tests/app/celery/test_scheduled_tasks.py | 20 ++++++++++++++++++++ 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/app/celery/scheduled_tasks.py b/app/celery/scheduled_tasks.py index e3fdb97f8..51618ae71 100644 --- a/app/celery/scheduled_tasks.py +++ b/app/celery/scheduled_tasks.py @@ -1,3 +1,5 @@ +import uuid + from datetime import ( datetime, timedelta @@ -8,7 +10,7 @@ from notifications_utils.statsd_decorators import statsd from sqlalchemy import and_ from sqlalchemy.exc import SQLAlchemyError -from app import notify_celery, zendesk_client +from app import cbc_proxy_client, notify_celery, zendesk_client from app.celery.tasks import ( process_job, get_recipient_csv_and_template_and_sender_id, @@ -291,3 +293,9 @@ def check_for_services_with_high_failure_rates_or_sending_to_tv_numbers(): message=message, ticket_type=zendesk_client.TYPE_INCIDENT ) + + +@notify_celery.task(name='send-canary-to-cbc-proxy') +def send_canary_to_cbc_proxy(): + identifier = str(uuid.uuid4()) + cbc_proxy_client.send_canary(identifier) diff --git a/app/config.py b/app/config.py index 06e561ccf..1ad9bcea7 100644 --- a/app/config.py +++ b/app/config.py @@ -303,6 +303,11 @@ class Config(object): 'schedule': crontab(hour=23, minute=00), 'options': {'queue': QueueNames.PERIODIC} }, + 'send-canary-to-cbc-proxy': { + 'task': 'send-canary-to-cbc-proxy', + 'schedule': timedelta(minutes=5), + 'options': {'queue': QueueNames.PERIODIC} + }, } CELERY_QUEUES = [] diff --git a/tests/app/celery/test_scheduled_tasks.py b/tests/app/celery/test_scheduled_tasks.py index 93be2a859..2c6bf3750 100644 --- a/tests/app/celery/test_scheduled_tasks.py +++ b/tests/app/celery/test_scheduled_tasks.py @@ -1,3 +1,4 @@ +import uuid from datetime import datetime, timedelta from unittest.mock import call @@ -555,3 +556,22 @@ def test_check_for_services_with_high_failure_rates_or_sending_to_tv_numbers( subject="[test] High failure rates for sms spotted for services", ticket_type='incident' ) + + +def test_send_canary_to_cbc_proxy_invokes_cbc_proxy_client( + mocker, +): + mock_send_canary = mocker.patch( + 'app.cbc_proxy_client.send_canary', + ) + + scheduled_tasks.send_canary_to_cbc_proxy() + + mock_send_canary.assert_called + # the 0th argument of the call to send_canary + identifier = mock_send_canary.mock_calls[0][1][0] + + try: + uuid.UUID(identifier) + except BaseException: + pytest.fail(f"{identifier} is not a valid uuid") From 5cff30aa473496adc4990a0d8d7864c3b80dce28 Mon Sep 17 00:00:00 2001 From: Pea Tyczynska Date: Tue, 27 Oct 2020 10:45:32 +0000 Subject: [PATCH 3/3] Send a log message informing that we are sending canary to CBC proxy --- app/celery/scheduled_tasks.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/celery/scheduled_tasks.py b/app/celery/scheduled_tasks.py index 51618ae71..ccac190ac 100644 --- a/app/celery/scheduled_tasks.py +++ b/app/celery/scheduled_tasks.py @@ -298,4 +298,6 @@ def check_for_services_with_high_failure_rates_or_sending_to_tv_numbers(): @notify_celery.task(name='send-canary-to-cbc-proxy') def send_canary_to_cbc_proxy(): identifier = str(uuid.uuid4()) + message = f"Sending a canary message to CBC proxy with ID {identifier}" + current_app.logger.info(message) cbc_proxy_client.send_canary(identifier)