From 6422a88c8c64d87a500df0723b7e7c30d283990a Mon Sep 17 00:00:00 2001 From: Pea Tyczynska Date: Mon, 1 Jun 2020 17:08:30 +0100 Subject: [PATCH 1/3] Stub SES email client to avoid hitting SES during load testing If we set an environment variable, we can stub out calls to SES and send them to our own stub app. If the environment variable is not set, things work as normal. To be used alongside https://github.com/alphagov/notifications-email-provider-stub --- app/__init__.py | 13 +++++++- app/clients/email/aws_ses_stub.py | 49 +++++++++++++++++++++++++++++++ app/config.py | 1 + 3 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 app/clients/email/aws_ses_stub.py diff --git a/app/__init__.py b/app/__init__.py index 2524bace0..4673a0ce3 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -21,6 +21,7 @@ from app.celery.celery import NotifyCelery from app.clients import Clients from app.clients.document_download import DocumentDownloadClient from app.clients.email.aws_ses import AwsSesClient +from app.clients.email.aws_ses_stub import AwsSesStubClient from app.clients.sms.firetext import FiretextClient from app.clients.sms.mmg import MMGClient from app.clients.performance_platform.performance_platform_client import PerformancePlatformClient @@ -49,6 +50,7 @@ notify_celery = NotifyCelery() firetext_client = FiretextClient() mmg_client = MMGClient() aws_ses_client = AwsSesClient() +aws_ses_stub_client = AwsSesStubClient() encryption = Encryption() zendesk_client = ZendeskClient() statsd_client = StatsdClient() @@ -81,13 +83,22 @@ def create_app(application): logging.init_app(application, statsd_client) firetext_client.init_app(application, statsd_client=statsd_client) mmg_client.init_app(application, statsd_client=statsd_client) + aws_ses_client.init_app(application.config['AWS_REGION'], statsd_client=statsd_client) + aws_ses_stub_client.init_app( + application.config['AWS_REGION'], + statsd_client=statsd_client, + stub_url=os.environ.get('SES_STUB_URL') + ) + # If a stub url is provided for SES, then use the stub client rather than the real SES boto client + email_clients = [aws_ses_client] if not os.environ.get('SES_STUB_URL') else [aws_ses_stub_client] + clients.init_app(sms_clients=[firetext_client, mmg_client], email_clients=email_clients) + notify_celery.init_app(application) encryption.init_app(application) redis_store.init_app(application) performance_platform_client.init_app(application) document_download_client.init_app(application) - clients.init_app(sms_clients=[firetext_client, mmg_client], email_clients=[aws_ses_client]) metrics.init_app(application) register_blueprint(application) diff --git a/app/clients/email/aws_ses_stub.py b/app/clients/email/aws_ses_stub.py new file mode 100644 index 000000000..0414b50b4 --- /dev/null +++ b/app/clients/email/aws_ses_stub.py @@ -0,0 +1,49 @@ +import json + +from flask import current_app +from requests import request +from time import monotonic + +from app.clients.email import (EmailClientException, EmailClient) + + +class AwsSesStubClientException(EmailClientException): + pass + + +class AwsSesStubClient(EmailClient): + def init_app(self, region, statsd_client, stub_url): + self.name = 'ses' + self.statsd_client = statsd_client + self.url = stub_url + + def get_name(self): + return self.name + + def send_email(self, + source, + to_addresses, + subject, + body, + html_body='', + reply_to_address=None): + try: + start_time = monotonic() + response = request( + "POST", + self.url, + data={"id": "dummy-data"}, + timeout=60 + ) + response.raise_for_status() + response_json = json.loads(response.text) + + except Exception as e: + self.statsd_client.incr("clients.ses_stub.error") + raise AwsSesStubClientException(str(e)) + else: + elapsed_time = monotonic() - start_time + current_app.logger.info("AWS SES stub request finished in {}".format(elapsed_time)) + self.statsd_client.timing("clients.ses_stub.request-time", elapsed_time) + self.statsd_client.incr("clients.ses_stub.success") + return response_json['MessageId'] diff --git a/app/config.py b/app/config.py index 77da4b843..e55933956 100644 --- a/app/config.py +++ b/app/config.py @@ -345,6 +345,7 @@ class Config(object): # these environment vars aren't defined in the manifest so to set them on paas use `cf set-env` MMG_URL = os.environ.get("MMG_URL", "https://api.mmg.co.uk/jsonv2a/api.php") FIRETEXT_URL = os.environ.get("FIRETEXT_URL", "https://www.firetext.co.uk/api/sendsms/json") + SES_STUB_URL = os.environ.get("SES_STUB_URL") AWS_REGION = 'eu-west-1' From 8b7a0b88cbdbadc3e94a0048c9e341f7261af598 Mon Sep 17 00:00:00 2001 From: Pea Tyczynska Date: Wed, 3 Jun 2020 12:34:04 +0100 Subject: [PATCH 2/3] Ensure that aws ses stub client is not run in production --- app/__init__.py | 4 ++-- app/config.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index 4673a0ce3..b33c78034 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -88,10 +88,10 @@ def create_app(application): aws_ses_stub_client.init_app( application.config['AWS_REGION'], statsd_client=statsd_client, - stub_url=os.environ.get('SES_STUB_URL') + stub_url=application.config['SES_STUB_URL'] ) # If a stub url is provided for SES, then use the stub client rather than the real SES boto client - email_clients = [aws_ses_client] if not os.environ.get('SES_STUB_URL') else [aws_ses_stub_client] + email_clients = [aws_ses_stub_client] if application.config['SES_STUB_URL'] else [aws_ses_client] clients.init_app(sms_clients=[firetext_client, mmg_client], email_clients=email_clients) notify_celery.init_app(application) diff --git a/app/config.py b/app/config.py index e55933956..a7aff994a 100644 --- a/app/config.py +++ b/app/config.py @@ -486,6 +486,7 @@ class Live(Config): PERFORMANCE_PLATFORM_ENABLED = True API_RATE_LIMIT_ENABLED = True CHECK_PROXY_HEADER = True + SES_STUB_URL = None CRONITOR_ENABLED = True From 7bc02ac26edf4941e25ce81895975638c887a969 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Thu, 4 Jun 2020 15:43:03 +0100 Subject: [PATCH 3/3] Add better logging to understand callback delivery cases In particular, we want to know how often callbacks arrive before the notification being persisted --- app/celery/process_ses_receipts_tasks.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/app/celery/process_ses_receipts_tasks.py b/app/celery/process_ses_receipts_tasks.py index ea4838e79..95e64b1f0 100644 --- a/app/celery/process_ses_receipts_tasks.py +++ b/app/celery/process_ses_receipts_tasks.py @@ -43,10 +43,14 @@ def process_ses_results(self, response): except NoResultFound: message_time = iso8601.parse_date(ses_message['mail']['timestamp']).replace(tzinfo=None) if datetime.utcnow() - message_time < timedelta(minutes=5): + current_app.logger.info( + f"notification not found for reference: {reference} (update to {notification_status}). " + f"Callback may have arrived before notification was persisted to the DB. Adding task to retry queue" + ) self.retry(queue=QueueNames.RETRY) else: current_app.logger.warning( - "notification not found for reference: {} (update to {})".format(reference, notification_status) + f"notification not found for reference: {reference} (update to {notification_status})" ) return