mirror of
https://github.com/GSA/notifications-api.git
synced 2025-12-22 16:31:15 -05:00
ac34fb9c05
Retry tasks if they fail to send a broadcast event. Note that each task tries the regular proxy and the failover proxy for that provider. This runs a bit differently than our other retries: Retry with exponential backoff. Our other tasks retry with a fixed delay of 5 minutes between tries. If we can't send a broadcast, we want to try immediately. So instead, implement an exponential backoff (1, 2, 4, 8, ... seconds delay). We can't delay for longer than 310 seconds due to visibility timeout settings in SQS, so cap the delay at that amount. Normally we give up retrying after a set amount of retries (often 4 hours). As broadcast content is much more important than normal notifications, we don't ever want to give up on sending them to phones... ...UNLESS WE DO! Sometimes we do want to give up sending a broadcast though! Broadcasts have an expiry time, when they stop showing up on peoples devices, so if that has passed then we don't need to send the broadcast out. Broadcast events can also be superceded by updates or cancels. Check that the event is the most recent event for that broadcast message, if not, give up, as we don't want to accidentally send out two conflicting events for the same message.
614 lines
17 KiB
Python
614 lines
17 KiB
Python
import json
|
|
import uuid
|
|
from collections import namedtuple
|
|
from datetime import datetime
|
|
from unittest.mock import Mock, call
|
|
|
|
import pytest
|
|
|
|
from app.clients.cbc_proxy import (
|
|
CBCProxyClient, CBCProxyRetryableException, CBCProxyEE, CBCProxyCanary, CBCProxyVodafone, CBCProxyThree, CBCProxyO2
|
|
)
|
|
from app.utils import DATETIME_FORMAT
|
|
|
|
EXAMPLE_AREAS = [{
|
|
'description': 'london',
|
|
'polygon': [
|
|
[51.12, -1.2],
|
|
[51.12, 1.2],
|
|
[51.74, 1.2],
|
|
[51.74, -1.2],
|
|
[51.12, -1.2],
|
|
],
|
|
}]
|
|
|
|
|
|
@pytest.fixture(scope='function')
|
|
def cbc_proxy_client(client, mocker):
|
|
client = CBCProxyClient()
|
|
current_app = mocker.Mock(config={
|
|
'CBC_PROXY_AWS_ACCESS_KEY_ID': 'cbc-proxy-aws-access-key-id',
|
|
'CBC_PROXY_AWS_SECRET_ACCESS_KEY': 'cbc-proxy-aws-secret-access-key',
|
|
'CBC_PROXY_ENABLED': True,
|
|
})
|
|
client.init_app(current_app)
|
|
return client
|
|
|
|
|
|
@pytest.fixture
|
|
def cbc_proxy_ee(cbc_proxy_client):
|
|
return cbc_proxy_client.get_proxy('ee')
|
|
|
|
@pytest.fixture
|
|
def cbc_proxy_vodafone(cbc_proxy_client):
|
|
return cbc_proxy_client.get_proxy('vodafone')
|
|
|
|
|
|
@pytest.mark.parametrize('provider_name, expected_provider_class', [
|
|
('ee', CBCProxyEE),
|
|
('three', CBCProxyThree),
|
|
('o2', CBCProxyO2),
|
|
('vodafone', CBCProxyVodafone),
|
|
('canary', CBCProxyCanary),
|
|
])
|
|
def test_cbc_proxy_client_returns_correct_client(provider_name, expected_provider_class):
|
|
mock_lambda = Mock()
|
|
cbc_proxy_client = CBCProxyClient()
|
|
cbc_proxy_client._lambda_client = mock_lambda
|
|
|
|
ret = cbc_proxy_client.get_proxy(provider_name)
|
|
|
|
assert type(ret) == expected_provider_class
|
|
assert ret._lambda_client == mock_lambda
|
|
|
|
|
|
def test_cbc_proxy_lambda_client_has_correct_region(cbc_proxy_ee):
|
|
assert cbc_proxy_ee._lambda_client._client_config.region_name == 'eu-west-2'
|
|
|
|
|
|
def test_cbc_proxy_lambda_client_has_correct_keys(cbc_proxy_ee):
|
|
key = cbc_proxy_ee._lambda_client._request_signer._credentials.access_key
|
|
secret = cbc_proxy_ee._lambda_client._request_signer._credentials.secret_key
|
|
|
|
assert key == 'cbc-proxy-aws-access-key-id'
|
|
assert secret == 'cbc-proxy-aws-secret-access-key'
|
|
|
|
|
|
@pytest.mark.parametrize('description, expected_language', (
|
|
('my-description', 'en-GB'),
|
|
('mŷ-description', 'cy-GB'),
|
|
))
|
|
@pytest.mark.parametrize('cbc', ['ee', 'three', 'o2'])
|
|
def test_cbc_proxy_one_2_many_create_and_send_invokes_function(
|
|
mocker,
|
|
cbc_proxy_client,
|
|
description,
|
|
cbc,
|
|
expected_language,
|
|
):
|
|
cbc_proxy = cbc_proxy_client.get_proxy(cbc)
|
|
|
|
identifier = 'my-identifier'
|
|
headline = 'my-headline'
|
|
|
|
sent = 'a-passed-through-sent-value'
|
|
expires = 'a-passed-through-expires-value'
|
|
|
|
ld_client_mock = mocker.patch.object(
|
|
cbc_proxy,
|
|
'_lambda_client',
|
|
create=True,
|
|
)
|
|
|
|
ld_client_mock.invoke.return_value = {
|
|
'StatusCode': 200,
|
|
}
|
|
|
|
cbc_proxy.create_and_send_broadcast(
|
|
identifier=identifier,
|
|
message_number='0000007b',
|
|
headline=headline,
|
|
description=description,
|
|
areas=EXAMPLE_AREAS,
|
|
sent=sent,
|
|
expires=expires,
|
|
channel="severe",
|
|
)
|
|
|
|
ld_client_mock.invoke.assert_called_once_with(
|
|
FunctionName=f'{cbc}-1-proxy',
|
|
InvocationType='RequestResponse',
|
|
Payload=mocker.ANY,
|
|
)
|
|
|
|
kwargs = ld_client_mock.invoke.mock_calls[0][-1]
|
|
payload_bytes = kwargs['Payload']
|
|
payload = json.loads(payload_bytes)
|
|
|
|
assert payload['identifier'] == identifier
|
|
assert 'message_number' not in payload
|
|
assert payload['message_format'] == 'cap'
|
|
assert payload['message_type'] == 'alert'
|
|
assert payload['headline'] == headline
|
|
assert payload['description'] == description
|
|
assert payload['areas'] == EXAMPLE_AREAS
|
|
assert payload['sent'] == sent
|
|
assert payload['expires'] == expires
|
|
assert payload['language'] == expected_language
|
|
assert payload['channel'] == 'severe'
|
|
|
|
|
|
@pytest.mark.parametrize('cbc', ['ee', 'three', 'o2'])
|
|
def test_cbc_proxy_one_2_many_cancel_invokes_function(mocker, cbc_proxy_client, cbc):
|
|
cbc_proxy = cbc_proxy_client.get_proxy(cbc)
|
|
|
|
identifier = 'my-identifier'
|
|
MockProviderMessage = namedtuple(
|
|
'BroadcastProviderMessage', ['id', 'message_number', 'created_at']
|
|
)
|
|
|
|
provider_messages = [
|
|
MockProviderMessage(uuid.uuid4(), '0000007b', datetime(2020, 12, 16)),
|
|
MockProviderMessage(uuid.uuid4(), '0000004e', datetime(2020, 12, 17))
|
|
]
|
|
sent = '2020-12-17 14:19:44.130585'
|
|
|
|
ld_client_mock = mocker.patch.object(
|
|
cbc_proxy,
|
|
'_lambda_client',
|
|
create=True,
|
|
)
|
|
|
|
ld_client_mock.invoke.return_value = {
|
|
'StatusCode': 200,
|
|
}
|
|
|
|
cbc_proxy.cancel_broadcast(
|
|
identifier=identifier,
|
|
message_number='00000050',
|
|
previous_provider_messages=provider_messages,
|
|
sent=sent
|
|
)
|
|
|
|
ld_client_mock.invoke.assert_called_once_with(
|
|
FunctionName=f'{cbc}-1-proxy',
|
|
InvocationType='RequestResponse',
|
|
Payload=mocker.ANY,
|
|
)
|
|
|
|
kwargs = ld_client_mock.invoke.mock_calls[0][-1]
|
|
payload_bytes = kwargs['Payload']
|
|
payload = json.loads(payload_bytes)
|
|
|
|
assert payload['identifier'] == identifier
|
|
assert 'message_number' not in payload
|
|
assert payload['message_format'] == 'cap'
|
|
assert payload['message_type'] == 'cancel'
|
|
assert payload['references'] == [
|
|
{
|
|
"message_id": str(provider_messages[0].id),
|
|
"sent": provider_messages[0].created_at.strftime(DATETIME_FORMAT)
|
|
},
|
|
{
|
|
"message_id": str(provider_messages[1].id),
|
|
"sent": provider_messages[1].created_at.strftime(DATETIME_FORMAT)
|
|
},
|
|
]
|
|
assert payload['sent'] == sent
|
|
|
|
|
|
@pytest.mark.parametrize('description, expected_language', (
|
|
('my-description', 'English'),
|
|
('mŷ-description', 'Welsh'),
|
|
))
|
|
def test_cbc_proxy_vodafone_create_and_send_invokes_function(
|
|
mocker,
|
|
cbc_proxy_vodafone,
|
|
description,
|
|
expected_language,
|
|
):
|
|
identifier = 'my-identifier'
|
|
headline = 'my-headline'
|
|
|
|
sent = 'a-passed-through-sent-value'
|
|
expires = 'a-passed-through-expires-value'
|
|
|
|
ld_client_mock = mocker.patch.object(
|
|
cbc_proxy_vodafone,
|
|
'_lambda_client',
|
|
create=True,
|
|
)
|
|
|
|
ld_client_mock.invoke.return_value = {
|
|
'StatusCode': 200,
|
|
}
|
|
|
|
cbc_proxy_vodafone.create_and_send_broadcast(
|
|
identifier=identifier,
|
|
message_number='0000007b',
|
|
headline=headline,
|
|
description=description,
|
|
areas=EXAMPLE_AREAS,
|
|
sent=sent,
|
|
expires=expires,
|
|
channel="test",
|
|
)
|
|
|
|
ld_client_mock.invoke.assert_called_once_with(
|
|
FunctionName='vodafone-1-proxy',
|
|
InvocationType='RequestResponse',
|
|
Payload=mocker.ANY,
|
|
)
|
|
|
|
kwargs = ld_client_mock.invoke.mock_calls[0][-1]
|
|
payload_bytes = kwargs['Payload']
|
|
payload = json.loads(payload_bytes)
|
|
|
|
assert payload['identifier'] == identifier
|
|
assert payload['message_number'] == '0000007b'
|
|
assert payload['message_format'] == 'ibag'
|
|
assert payload['message_type'] == 'alert'
|
|
assert payload['headline'] == headline
|
|
assert payload['description'] == description
|
|
assert payload['areas'] == EXAMPLE_AREAS
|
|
assert payload['sent'] == sent
|
|
assert payload['expires'] == expires
|
|
assert payload['language'] == expected_language
|
|
assert payload['channel'] == 'test'
|
|
|
|
|
|
def test_cbc_proxy_vodafone_cancel_invokes_function(mocker, cbc_proxy_vodafone):
|
|
identifier = 'my-identifier'
|
|
MockProviderMessage = namedtuple(
|
|
'BroadcastProviderMessage',
|
|
['id', 'message_number', 'created_at']
|
|
)
|
|
|
|
provider_messages = [
|
|
MockProviderMessage(uuid.uuid4(), 78, datetime(2020, 12, 16)),
|
|
MockProviderMessage(uuid.uuid4(), 123, datetime(2020, 12, 17))
|
|
]
|
|
sent = '2020-12-18 14:19:44.130585'
|
|
|
|
ld_client_mock = mocker.patch.object(
|
|
cbc_proxy_vodafone,
|
|
'_lambda_client',
|
|
create=True,
|
|
)
|
|
|
|
ld_client_mock.invoke.return_value = {
|
|
'StatusCode': 200,
|
|
}
|
|
|
|
cbc_proxy_vodafone.cancel_broadcast(
|
|
identifier=identifier,
|
|
message_number='00000050',
|
|
previous_provider_messages=provider_messages,
|
|
sent=sent
|
|
)
|
|
|
|
ld_client_mock.invoke.assert_called_once_with(
|
|
FunctionName='vodafone-1-proxy',
|
|
InvocationType='RequestResponse',
|
|
Payload=mocker.ANY,
|
|
)
|
|
|
|
kwargs = ld_client_mock.invoke.mock_calls[0][-1]
|
|
payload_bytes = kwargs['Payload']
|
|
payload = json.loads(payload_bytes)
|
|
|
|
assert payload['identifier'] == identifier
|
|
assert payload['message_number'] == '00000050'
|
|
assert payload['message_format'] == 'ibag'
|
|
assert payload['message_type'] == 'cancel'
|
|
assert payload['references'] == [
|
|
{
|
|
"message_id": str(provider_messages[0].id),
|
|
"message_number": '0000004e',
|
|
"sent": provider_messages[0].created_at.strftime(DATETIME_FORMAT)
|
|
},
|
|
{
|
|
"message_id": str(provider_messages[1].id),
|
|
"message_number": '0000007b',
|
|
"sent": provider_messages[1].created_at.strftime(DATETIME_FORMAT)
|
|
},
|
|
]
|
|
assert payload['sent'] == sent
|
|
|
|
|
|
@pytest.mark.parametrize('cbc', ['ee', 'vodafone', 'three', 'o2'])
|
|
def test_cbc_proxy_will_failover_to_second_lambda_if_function_error(
|
|
mocker,
|
|
cbc_proxy_client,
|
|
cbc
|
|
):
|
|
cbc_proxy = cbc_proxy_client.get_proxy(cbc)
|
|
|
|
ld_client_mock = mocker.patch.object(
|
|
cbc_proxy,
|
|
'_lambda_client',
|
|
create=True,
|
|
)
|
|
|
|
ld_client_mock.invoke.side_effect = [
|
|
{
|
|
'StatusCode': 200,
|
|
'FunctionError': 'Handled',
|
|
'Payload': {
|
|
"errorMessage": "",
|
|
"errorType": "CBCNewConnectionError"
|
|
}
|
|
},
|
|
{
|
|
'StatusCode': 200
|
|
}
|
|
]
|
|
|
|
cbc_proxy.create_and_send_broadcast(
|
|
identifier='my-identifier',
|
|
message_number='0000007b',
|
|
headline='my-headline',
|
|
description='test-description',
|
|
areas=EXAMPLE_AREAS,
|
|
sent='a-passed-through-sent-value',
|
|
expires='a-passed-through-expires-value',
|
|
channel="severe",
|
|
)
|
|
|
|
assert ld_client_mock.invoke.call_args_list == [
|
|
call(
|
|
FunctionName=f'{cbc}-1-proxy',
|
|
InvocationType='RequestResponse',
|
|
Payload=mocker.ANY,
|
|
),
|
|
call(
|
|
FunctionName=f'{cbc}-2-proxy',
|
|
InvocationType='RequestResponse',
|
|
Payload=mocker.ANY,
|
|
)
|
|
]
|
|
|
|
|
|
@pytest.mark.parametrize('cbc', ['ee', 'vodafone', 'three', 'o2'])
|
|
def test_cbc_proxy_will_failover_to_second_lambda_if_invoke_error(
|
|
mocker,
|
|
cbc_proxy_client,
|
|
cbc
|
|
):
|
|
cbc_proxy = cbc_proxy_client.get_proxy(cbc)
|
|
|
|
ld_client_mock = mocker.patch.object(
|
|
cbc_proxy,
|
|
'_lambda_client',
|
|
create=True,
|
|
)
|
|
|
|
ld_client_mock.invoke.side_effect = [
|
|
{
|
|
'StatusCode': 400
|
|
},
|
|
{
|
|
'StatusCode': 200
|
|
}
|
|
]
|
|
|
|
cbc_proxy.create_and_send_broadcast(
|
|
identifier='my-identifier',
|
|
message_number='0000007b',
|
|
headline='my-headline',
|
|
description='test-description',
|
|
areas=EXAMPLE_AREAS,
|
|
sent='a-passed-through-sent-value',
|
|
expires='a-passed-through-expires-value',
|
|
channel="test",
|
|
)
|
|
|
|
assert ld_client_mock.invoke.call_args_list == [
|
|
call(
|
|
FunctionName=f'{cbc}-1-proxy',
|
|
InvocationType='RequestResponse',
|
|
Payload=mocker.ANY,
|
|
),
|
|
call(
|
|
FunctionName=f'{cbc}-2-proxy',
|
|
InvocationType='RequestResponse',
|
|
Payload=mocker.ANY,
|
|
)
|
|
]
|
|
|
|
|
|
@pytest.mark.parametrize('cbc', ['ee', 'vodafone', 'three', 'o2'])
|
|
def test_cbc_proxy_create_and_send_tries_failover_lambda_on_invoke_error_and_raises_if_both_invoke_error(
|
|
mocker, cbc_proxy_client, cbc
|
|
):
|
|
cbc_proxy = cbc_proxy_client.get_proxy(cbc)
|
|
|
|
ld_client_mock = mocker.patch.object(
|
|
cbc_proxy,
|
|
'_lambda_client',
|
|
create=True,
|
|
)
|
|
|
|
ld_client_mock.invoke.return_value = {
|
|
'StatusCode': 400,
|
|
}
|
|
|
|
with pytest.raises(CBCProxyRetryableException) as e:
|
|
cbc_proxy.create_and_send_broadcast(
|
|
identifier='my-identifier',
|
|
message_number='0000007b',
|
|
headline='my-headline',
|
|
description='my-description',
|
|
areas=EXAMPLE_AREAS,
|
|
sent='a-passed-through-sent-value',
|
|
expires='a-passed-through-expires-value',
|
|
channel="test",
|
|
)
|
|
|
|
assert e.match(f'Lambda failed for both {cbc}-1-proxy and {cbc}-2-proxy')
|
|
|
|
assert ld_client_mock.invoke.call_args_list == [
|
|
call(
|
|
FunctionName=f'{cbc}-1-proxy',
|
|
InvocationType='RequestResponse',
|
|
Payload=mocker.ANY,
|
|
),
|
|
call(
|
|
FunctionName=f'{cbc}-2-proxy',
|
|
InvocationType='RequestResponse',
|
|
Payload=mocker.ANY,
|
|
)
|
|
]
|
|
|
|
|
|
@pytest.mark.parametrize('cbc', ['ee', 'vodafone', 'three', 'o2'])
|
|
def test_cbc_proxy_create_and_send_tries_failover_lambda_on_function_error_and_raises_if_both_function_error(
|
|
mocker, cbc_proxy_client, cbc
|
|
):
|
|
cbc_proxy = cbc_proxy_client.get_proxy(cbc)
|
|
|
|
ld_client_mock = mocker.patch.object(
|
|
cbc_proxy,
|
|
'_lambda_client',
|
|
create=True,
|
|
)
|
|
|
|
ld_client_mock.invoke.return_value = {
|
|
'StatusCode': 200,
|
|
'FunctionError': 'something',
|
|
'Payload': {
|
|
'errorMessage': 'some message',
|
|
'errorType': 'SomeErrorType'
|
|
}
|
|
}
|
|
|
|
with pytest.raises(CBCProxyRetryableException) as e:
|
|
cbc_proxy.create_and_send_broadcast(
|
|
identifier='my-identifier',
|
|
message_number='0000007b',
|
|
headline='my-headline',
|
|
description='my-description',
|
|
areas=EXAMPLE_AREAS,
|
|
sent='a-passed-through-sent-value',
|
|
expires='a-passed-through-expires-value',
|
|
channel="severe",
|
|
)
|
|
|
|
assert e.match(f'Lambda failed for both {cbc}-1-proxy and {cbc}-2-proxy')
|
|
|
|
assert ld_client_mock.invoke.call_args_list == [
|
|
call(
|
|
FunctionName=f'{cbc}-1-proxy',
|
|
InvocationType='RequestResponse',
|
|
Payload=mocker.ANY,
|
|
),
|
|
call(
|
|
FunctionName=f'{cbc}-2-proxy',
|
|
InvocationType='RequestResponse',
|
|
Payload=mocker.ANY,
|
|
)
|
|
]
|
|
|
|
|
|
def test_cbc_proxy_send_canary_invokes_function(mocker, cbc_proxy_client):
|
|
identifier = str(uuid.uuid4())
|
|
|
|
canary_client = cbc_proxy_client.get_proxy('canary')
|
|
|
|
ld_client_mock = mocker.patch.object(
|
|
canary_client,
|
|
'_lambda_client',
|
|
create=True,
|
|
)
|
|
|
|
ld_client_mock.invoke.return_value = {
|
|
'StatusCode': 200,
|
|
}
|
|
|
|
canary_client.send_canary(
|
|
identifier=identifier,
|
|
)
|
|
|
|
ld_client_mock.invoke.assert_called_once_with(
|
|
FunctionName='canary',
|
|
InvocationType='RequestResponse',
|
|
Payload=mocker.ANY,
|
|
)
|
|
|
|
kwargs = ld_client_mock.invoke.mock_calls[0][-1]
|
|
payload_bytes = kwargs['Payload']
|
|
payload = json.loads(payload_bytes)
|
|
|
|
assert payload['identifier'] == identifier
|
|
|
|
|
|
@pytest.mark.parametrize('cbc', ['ee', 'three', 'o2'])
|
|
def test_cbc_proxy_one_2_many_send_link_test_invokes_function(mocker, cbc_proxy_client, cbc):
|
|
cbc_proxy = cbc_proxy_client.get_proxy(cbc)
|
|
|
|
identifier = str(uuid.uuid4())
|
|
|
|
ld_client_mock = mocker.patch.object(
|
|
cbc_proxy,
|
|
'_lambda_client',
|
|
create=True,
|
|
)
|
|
|
|
ld_client_mock.invoke.return_value = {
|
|
'StatusCode': 200,
|
|
}
|
|
|
|
cbc_proxy.send_link_test(
|
|
identifier=identifier,
|
|
sequential_number='0000007b',
|
|
)
|
|
|
|
ld_client_mock.invoke.assert_called_once_with(
|
|
FunctionName=f'{cbc}-1-proxy',
|
|
InvocationType='RequestResponse',
|
|
Payload=mocker.ANY,
|
|
)
|
|
|
|
kwargs = ld_client_mock.invoke.mock_calls[0][-1]
|
|
payload_bytes = kwargs['Payload']
|
|
payload = json.loads(payload_bytes)
|
|
|
|
assert payload['identifier'] == identifier
|
|
assert payload['message_type'] == 'test'
|
|
assert 'message_number' not in payload
|
|
assert payload['message_format'] == 'cap'
|
|
|
|
|
|
def test_cbc_proxy_vodafone_send_link_test_invokes_function(mocker, cbc_proxy_vodafone):
|
|
identifier = str(uuid.uuid4())
|
|
|
|
ld_client_mock = mocker.patch.object(
|
|
cbc_proxy_vodafone,
|
|
'_lambda_client',
|
|
create=True,
|
|
)
|
|
|
|
ld_client_mock.invoke.return_value = {
|
|
'StatusCode': 200,
|
|
}
|
|
|
|
cbc_proxy_vodafone.send_link_test(
|
|
identifier=identifier,
|
|
sequential_number='0000007b',
|
|
)
|
|
|
|
ld_client_mock.invoke.assert_called_once_with(
|
|
FunctionName='vodafone-1-proxy',
|
|
InvocationType='RequestResponse',
|
|
Payload=mocker.ANY,
|
|
)
|
|
|
|
kwargs = ld_client_mock.invoke.mock_calls[0][-1]
|
|
payload_bytes = kwargs['Payload']
|
|
payload = json.loads(payload_bytes)
|
|
|
|
assert payload['identifier'] == identifier
|
|
assert payload['message_type'] == 'test'
|
|
assert payload['message_number'] == '0000007b'
|
|
assert payload['message_format'] == 'ibag'
|