mirror of
https://github.com/GSA/notifications-api.git
synced 2026-09-02 17:48:36 -04:00
using lots of CPU by tracing the application to gather some data Alternatively we could take a stab in the dark, which is what this commit is doing. I have the hypothesis that we are not re-using TCP connections using HTTP keepalive Refer to https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Keep-Alive This means we are renegotiating the TLS connection every time we want to send an SMS. When we are sending lots of SMS messages then this will do a lot of crypto handshaking which is expensive (in terms of CPU) ie send_sms calls request() which creates a new tcp connection and a new TLS handshake When you use request.Session() to create a session it uses urllib3's connection pooling, I've arbitrarily chosen 32 connections per pool (with a default number of pools = 10) ie init_app creates a session which has underlying connection pools send_sms claims a connection from the pool and uses it to create or re-use an existing TLS connection Sessions are usually not great because they share data like cookies, but when calling an API this is fine, or at least it is probably worth canarying Another way of re-using connections is by running HAProxy or similar as a side-car proxy, which proxies to the API. send_sms would make a local TCP connection to HAProxy which proxies to the MMG or Firetext API via TLS, adding the Connection: keep-alive header This command can be used to see how many TLS handshakes your computer is doing, with some false positives: tcpdump -n "tcp port 443 and (tcp[((tcp[12] & 0xf0) >> 2)] = 0x16)" (alternatively we could just instrument the code /shrug) Signed-off-by: toby lorne <toby@toby.codes>
158 lines
5.8 KiB
Python
158 lines
5.8 KiB
Python
import boto3
|
|
import botocore
|
|
from flask import current_app
|
|
from time import monotonic
|
|
from notifications_utils.recipients import InvalidEmailError
|
|
|
|
from app.clients import STATISTICS_DELIVERED, STATISTICS_FAILURE
|
|
from app.clients.email import EmailClient, EmailClientException, EmailClientNonRetryableException
|
|
|
|
ses_response_map = {
|
|
'Permanent': {
|
|
"message": 'Hard bounced',
|
|
"success": False,
|
|
"notification_status": 'permanent-failure',
|
|
"notification_statistics_status": STATISTICS_FAILURE
|
|
},
|
|
'Temporary': {
|
|
"message": 'Soft bounced',
|
|
"success": False,
|
|
"notification_status": 'temporary-failure',
|
|
"notification_statistics_status": STATISTICS_FAILURE
|
|
},
|
|
'Delivery': {
|
|
"message": 'Delivered',
|
|
"success": True,
|
|
"notification_status": 'delivered',
|
|
"notification_statistics_status": STATISTICS_DELIVERED
|
|
},
|
|
'Complaint': {
|
|
"message": 'Complaint',
|
|
"success": True,
|
|
"notification_status": 'delivered',
|
|
"notification_statistics_status": STATISTICS_DELIVERED
|
|
}
|
|
}
|
|
|
|
|
|
def get_aws_responses(status):
|
|
return ses_response_map[status]
|
|
|
|
|
|
class AwsSesClientException(EmailClientException):
|
|
pass
|
|
|
|
|
|
class AwsSesClientThrottlingSendRateException(AwsSesClientException):
|
|
pass
|
|
|
|
|
|
class AwsSesClient(EmailClient):
|
|
'''
|
|
Amazon SES email client.
|
|
'''
|
|
|
|
def init_app(self, region, statsd_client, *args, **kwargs):
|
|
self._client = boto3.client('ses', region_name=region)
|
|
super(AwsSesClient, self).__init__(*args, **kwargs)
|
|
self.name = 'ses'
|
|
self.statsd_client = statsd_client
|
|
|
|
# events are generally undocumented, but some that might be of interest are:
|
|
# before-call, after-call, after-call-error, request-created, response-received
|
|
self._client.meta.events.register('request-created.ses.SendEmail', self.ses_request_created_hook)
|
|
self._client.meta.events.register('response-received.ses.SendEmail', self.ses_response_received_hook)
|
|
self._client.meta.events.register('before-call.ses', self.ses_inject_connection_header)
|
|
|
|
def ses_request_created_hook(self, **kwargs):
|
|
# request created may be called multiple times if the request auto-retries. We want to count all these as the
|
|
# same request for timing purposes, so only reset the start time if it was cleared completely
|
|
if self.ses_start_time == 0:
|
|
self.ses_start_time = monotonic()
|
|
|
|
def ses_response_received_hook(self, **kwargs):
|
|
# response received may be called multiple times if the request auto-retries, however, we want to count the last
|
|
# time it triggers for timing purposes, so always reset the elapsed time
|
|
self.ses_elapsed_time = monotonic() - self.ses_start_time
|
|
|
|
def get_name(self):
|
|
return self.name
|
|
|
|
def ses_inject_connection_header(self, params, **kwargs):
|
|
# keep underlying TLS connection open, so we do not spend lots of CPU
|
|
# and network time renegotiating TLS
|
|
params['headers']['Connection'] = 'Keep-Alive'
|
|
|
|
def send_email(self,
|
|
source,
|
|
to_addresses,
|
|
subject,
|
|
body,
|
|
html_body='',
|
|
reply_to_address=None):
|
|
self.ses_elapsed_time = 0
|
|
self.ses_start_time = 0
|
|
try:
|
|
if isinstance(to_addresses, str):
|
|
to_addresses = [to_addresses]
|
|
|
|
reply_to_addresses = [reply_to_address] if reply_to_address else []
|
|
|
|
body = {
|
|
'Text': {'Data': body}
|
|
}
|
|
|
|
if html_body:
|
|
body.update({
|
|
'Html': {'Data': html_body}
|
|
})
|
|
|
|
start_time = monotonic()
|
|
response = self._client.send_email(
|
|
Source=source,
|
|
Destination={
|
|
'ToAddresses': [punycode_encode_email(addr) for addr in to_addresses],
|
|
'CcAddresses': [],
|
|
'BccAddresses': []
|
|
},
|
|
Message={
|
|
'Subject': {
|
|
'Data': subject,
|
|
},
|
|
'Body': body
|
|
},
|
|
ReplyToAddresses=[punycode_encode_email(addr) for addr in reply_to_addresses]
|
|
)
|
|
except botocore.exceptions.ClientError as e:
|
|
self.statsd_client.incr("clients.ses.error")
|
|
|
|
# http://docs.aws.amazon.com/ses/latest/DeveloperGuide/api-error-codes.html
|
|
if e.response['Error']['Code'] == 'InvalidParameterValue':
|
|
raise EmailClientNonRetryableException(e.response['Error']['Message'])
|
|
elif (
|
|
e.response['Error']['Code'] == 'Throttling'
|
|
and e.response['Error']['Message'] == 'Maximum sending rate exceeded.'
|
|
):
|
|
raise AwsSesClientThrottlingSendRateException(str(e))
|
|
else:
|
|
self.statsd_client.incr("clients.ses.error")
|
|
raise AwsSesClientException(str(e))
|
|
except Exception as e:
|
|
self.statsd_client.incr("clients.ses.error")
|
|
raise AwsSesClientException(str(e))
|
|
else:
|
|
elapsed_time = monotonic() - start_time
|
|
current_app.logger.info("AWS SES request finished in {}".format(elapsed_time))
|
|
self.statsd_client.timing("clients.ses.request-time", elapsed_time)
|
|
if self.ses_elapsed_time != 0:
|
|
self.statsd_client.timing("clients.ses.raw-request-time", self.ses_elapsed_time)
|
|
self.statsd_client.incr("clients.ses.success")
|
|
return response['MessageId']
|
|
|
|
|
|
def punycode_encode_email(email_address):
|
|
# only the hostname should ever be punycode encoded.
|
|
local, hostname = email_address.split('@')
|
|
return '{}@{}'.format(local, hostname.encode('idna').decode('utf-8'))
|
|
|