Merge pull request #3137 from alphagov/revert-revert-revert

Bring back retry logic

app/clients/cbc_proxy.py

@@ -2,6 +2,7 @@ import json
 from abc import ABC, abstractmethod
 
 import boto3
+import botocore
 from flask import current_app
 from notifications_utils.template import non_gsm_characters
 
@@ -25,7 +26,11 @@ from app.utils import DATETIME_FORMAT, format_sequential_number
 #    the preceeding Alert message in the previous_provider_messages field
 
 
-class CBCProxyException(Exception):
+class CBCProxyFatalException(Exception):
+    pass
+
+
+class CBCProxyRetryableException(Exception):
     pass
 
 
@@ -115,19 +120,24 @@ class CBCProxyClientBase(ABC):
         if not result:
             failover_result = self._invoke_lambda(self.failover_lambda_name, payload)
             if not failover_result:
-                raise CBCProxyException(f'Lambda failed for both {self.lambda_name} and {self.failover_lambda_name}')
+                raise CBCProxyRetryableException(
+                    f'Lambda failed for both {self.lambda_name} and {self.failover_lambda_name}'
+                )
 
         return result
 
     def _invoke_lambda(self, lambda_name, payload):
         payload_bytes = bytes(json.dumps(payload), encoding='utf8')
-        current_app.logger.info(f"Calling lambda {lambda_name}")
-        result = self._lambda_client.invoke(
-            FunctionName=lambda_name,
-            InvocationType='RequestResponse',
-            Payload=payload_bytes,
-        )
-        current_app.logger.info(f"Finished calling lambda {lambda_name}")
+        try:
+            result = self._lambda_client.invoke(
+                FunctionName=lambda_name,
+                InvocationType='RequestResponse',
+                Payload=payload_bytes,
+            )
+        except botocore.exceptions.ClientError as exc:
+            current_app.logger.exception(f'Boto ClientError calling lambda {lambda_name}')
+            success = False
+            return success
 
         if result['StatusCode'] > 299:
             current_app.logger.info(