catch boto exceptions

these will happen if, for example, you have issues connecting to AWS or
permission issues.

Still failover if we get one of these exceptions, as I think it might be
possible to have a problem only related to one of the lambdas.

app/clients/cbc_proxy.py

@@ -2,6 +2,7 @@ import json
 from abc import ABC, abstractmethod
 
 import boto3
+import botocore
 from flask import current_app
 from notifications_utils.template import non_gsm_characters
 
@@ -127,13 +128,16 @@ class CBCProxyClientBase(ABC):
 
     def _invoke_lambda(self, lambda_name, payload):
         payload_bytes = bytes(json.dumps(payload), encoding='utf8')
-        current_app.logger.info(f"Calling lambda {lambda_name}")
-        result = self._lambda_client.invoke(
-            FunctionName=lambda_name,
-            InvocationType='RequestResponse',
-            Payload=payload_bytes,
-        )
-        current_app.logger.info(f"Finished calling lambda {lambda_name}")
+        try:
+            result = self._lambda_client.invoke(
+                FunctionName=lambda_name,
+                InvocationType='RequestResponse',
+                Payload=payload_bytes,
+            )
+        except botocore.exceptions.ClientError as exc:
+            current_app.logger.exception(f'Boto ClientError calling lambda {lambda_name}')
+            success = False
+            return success
 
         if result['StatusCode'] > 299:
             current_app.logger.info(

tests/app/clients/test_cbc_proxy.py

@@ -4,6 +4,7 @@ from collections import namedtuple
 from datetime import datetime
 from unittest.mock import Mock, call
 
+from botocore.exceptions import ClientError as BotoClientError
 import pytest
 
 from app.clients.cbc_proxy import (
@@ -316,6 +317,50 @@ def test_cbc_proxy_vodafone_cancel_invokes_function(mocker, cbc_proxy_vodafone):
     assert payload['sent'] == sent
 
 
+@pytest.mark.parametrize('cbc', ['ee', 'vodafone', 'three', 'o2'])
+def test_cbc_proxy_will_failover_to_second_lambda_if_boto_client_error(
+    mocker,
+    cbc_proxy_client,
+    cbc
+):
+    cbc_proxy = cbc_proxy_client.get_proxy(cbc)
+
+    ld_client_mock = mocker.patch.object(
+        cbc_proxy,
+        '_lambda_client',
+        create=True,
+    )
+
+    ld_client_mock.invoke.side_effect = BotoClientError({}, 'error')
+
+    with pytest.raises(CBCProxyRetryableException) as e:
+        cbc_proxy.create_and_send_broadcast(
+            identifier='my-identifier',
+            message_number='0000007b',
+            headline='my-headline',
+            description='test-description',
+            areas=EXAMPLE_AREAS,
+            sent='a-passed-through-sent-value',
+            expires='a-passed-through-expires-value',
+            channel="severe",
+        )
+
+    assert e.match(f'Lambda failed for both {cbc}-1-proxy and {cbc}-2-proxy')
+
+    assert ld_client_mock.invoke.call_args_list == [
+        call(
+            FunctionName=f'{cbc}-1-proxy',
+            InvocationType='RequestResponse',
+            Payload=mocker.ANY,
+        ),
+        call(
+            FunctionName=f'{cbc}-2-proxy',
+            InvocationType='RequestResponse',
+            Payload=mocker.ANY,
+        )
+    ]
+
+
 @pytest.mark.parametrize('cbc', ['ee', 'vodafone', 'three', 'o2'])
 def test_cbc_proxy_will_failover_to_second_lambda_if_function_error(
     mocker,