Merge pull request #2862 from alphagov/ses-client-stub

Stub SES email client to avoid hitting SES during load testing

app/__init__.py

@@ -21,6 +21,7 @@ from app.celery.celery import NotifyCelery
 from app.clients import Clients
 from app.clients.document_download import DocumentDownloadClient
 from app.clients.email.aws_ses import AwsSesClient
+from app.clients.email.aws_ses_stub import AwsSesStubClient
 from app.clients.sms.firetext import FiretextClient
 from app.clients.sms.mmg import MMGClient
 from app.clients.performance_platform.performance_platform_client import PerformancePlatformClient
@@ -49,6 +50,7 @@ notify_celery = NotifyCelery()
 firetext_client = FiretextClient()
 mmg_client = MMGClient()
 aws_ses_client = AwsSesClient()
+aws_ses_stub_client = AwsSesStubClient()
 encryption = Encryption()
 zendesk_client = ZendeskClient()
 statsd_client = StatsdClient()
@@ -81,13 +83,22 @@ def create_app(application):
     logging.init_app(application, statsd_client)
     firetext_client.init_app(application, statsd_client=statsd_client)
     mmg_client.init_app(application, statsd_client=statsd_client)
+
     aws_ses_client.init_app(application.config['AWS_REGION'], statsd_client=statsd_client)
+    aws_ses_stub_client.init_app(
+        application.config['AWS_REGION'],
+        statsd_client=statsd_client,
+        stub_url=application.config['SES_STUB_URL']
+    )
+    # If a stub url is provided for SES, then use the stub client rather than the real SES boto client
+    email_clients = [aws_ses_stub_client] if application.config['SES_STUB_URL'] else [aws_ses_client]
+    clients.init_app(sms_clients=[firetext_client, mmg_client], email_clients=email_clients)
+
     notify_celery.init_app(application)
     encryption.init_app(application)
     redis_store.init_app(application)
     performance_platform_client.init_app(application)
     document_download_client.init_app(application)
-    clients.init_app(sms_clients=[firetext_client, mmg_client], email_clients=[aws_ses_client])
     metrics.init_app(application)
 
     register_blueprint(application)

app/celery/process_ses_receipts_tasks.py

@@ -43,10 +43,14 @@ def process_ses_results(self, response):
         except NoResultFound:
             message_time = iso8601.parse_date(ses_message['mail']['timestamp']).replace(tzinfo=None)
             if datetime.utcnow() - message_time < timedelta(minutes=5):
+                current_app.logger.info(
+                    f"notification not found for reference: {reference} (update to {notification_status}). "
+                    f"Callback may have arrived before notification was persisted to the DB. Adding task to retry queue"
+                )
                 self.retry(queue=QueueNames.RETRY)
             else:
                 current_app.logger.warning(
-                    "notification not found for reference: {} (update to {})".format(reference, notification_status)
+                    f"notification not found for reference: {reference} (update to {notification_status})"
                 )
             return
 

app/clients/email/aws_ses_stub.py

@@ -0,0 +1,49 @@
+import json
+
+from flask import current_app
+from requests import request
+from time import monotonic
+
+from app.clients.email import (EmailClientException, EmailClient)
+
+
+class AwsSesStubClientException(EmailClientException):
+    pass
+
+
+class AwsSesStubClient(EmailClient):
+    def init_app(self, region, statsd_client, stub_url):
+        self.name = 'ses'
+        self.statsd_client = statsd_client
+        self.url = stub_url
+
+    def get_name(self):
+        return self.name
+
+    def send_email(self,
+                   source,
+                   to_addresses,
+                   subject,
+                   body,
+                   html_body='',
+                   reply_to_address=None):
+        try:
+            start_time = monotonic()
+            response = request(
+                "POST",
+                self.url,
+                data={"id": "dummy-data"},
+                timeout=60
+            )
+            response.raise_for_status()
+            response_json = json.loads(response.text)
+
+        except Exception as e:
+            self.statsd_client.incr("clients.ses_stub.error")
+            raise AwsSesStubClientException(str(e))
+        else:
+            elapsed_time = monotonic() - start_time
+            current_app.logger.info("AWS SES stub request finished in {}".format(elapsed_time))
+            self.statsd_client.timing("clients.ses_stub.request-time", elapsed_time)
+            self.statsd_client.incr("clients.ses_stub.success")
+            return response_json['MessageId']

app/config.py

@@ -345,6 +345,7 @@ class Config(object):
     # these environment vars aren't defined in the manifest so to set them on paas use `cf set-env`
     MMG_URL = os.environ.get("MMG_URL", "https://api.mmg.co.uk/jsonv2a/api.php")
     FIRETEXT_URL = os.environ.get("FIRETEXT_URL", "https://www.firetext.co.uk/api/sendsms/json")
+    SES_STUB_URL = os.environ.get("SES_STUB_URL")
 
     AWS_REGION = 'eu-west-1'
 
@@ -485,6 +486,7 @@ class Live(Config):
     PERFORMANCE_PLATFORM_ENABLED = True
     API_RATE_LIMIT_ENABLED = True
     CHECK_PROXY_HEADER = True
+    SES_STUB_URL = None
 
     CRONITOR_ENABLED = True