remove outdated validatesns library and replace with maintainable code

Makefile

@@ -71,6 +71,7 @@ test: ## Run tests
 freeze-requirements: ## Pin all requirements including sub dependencies into requirements.txt
 	pip install --upgrade pip-tools
 	pip-compile requirements.in
+	pip3 install -r requirements.txt
 
 .PHONY: audit
 audit:

app/celery/process_ses_receipts_tasks.py

@@ -4,21 +4,20 @@ from json import decoder
 
 import iso8601
 import requests
-import validatesns
 from celery.exceptions import Retry
 from flask import Blueprint, current_app, json, jsonify, request
-from notifications_utils.statsd_decorators import statsd
 from sqlalchemy.orm.exc import NoResultFound
 
-from app import notify_celery, redis_store, statsd_client
+from app import notify_celery, statsd_client
+from app.celery.validate_sns import valid_sns_message
 from app.config import QueueNames
 from app.dao import notifications_dao
 from app.errors import InvalidRequest, register_errors
 from app.models import NOTIFICATION_PENDING, NOTIFICATION_SENDING
 from app.notifications.notifications_ses_callback import (
     _check_and_queue_complaint_callback_task,
-    _determine_notification_bounce_type,
     check_and_queue_callback_task,
+    determine_notification_bounce_type,
     get_aws_responses,
     handle_complaint,
 )
@@ -26,7 +25,6 @@ from app.notifications.notifications_ses_callback import (
 ses_callback_blueprint = Blueprint('notifications_ses_callback', __name__)
 
 register_errors(ses_callback_blueprint)
-
 class SNSMessageType(enum.Enum):
     SubscriptionConfirmation = 'SubscriptionConfirmation'
     Notification = 'Notification'
@@ -44,14 +42,6 @@ def verify_message_type(message_type: str):
         raise InvalidMessageTypeException(f'{message_type} is not a valid message type.')
 
 
-def get_certificate(url):
-    res = redis_store.get(url)
-    if res is not None:
-        return res
-    res = requests.get(url).content
-    redis_store.set(url, res, ex=60 * 60)  # 60 minutes
-    return res
-
 # 400 counts as a permanent failure so SNS will not retry.
 # 500 counts as a failed delivery attempt so SNS will retry.
 # See https://docs.aws.amazon.com/sns/latest/dg/DeliveryPolicies.html#DeliveryPolicies
@@ -62,35 +52,53 @@ def get_certificate(url):
 def sns_callback_handler():
     message_type = request.headers.get('x-amz-sns-message-type')
     try:
+        print("validating message type")
         verify_message_type(message_type)
     except InvalidMessageTypeException:
+        current_app.logger.exception(f"Response headers: {request.headers}\nResponse data: {request.data}")
         raise InvalidRequest("SES-SNS callback failed: invalid message type", 400)
 
     try:
-        message = json.loads(request.data)
+        print("loading message")
+        message = json.loads(request.data.decode('utf-8'))
     except decoder.JSONDecodeError:
+        current_app.logger.exception(f"Response headers: {request.headers}\nResponse data: {request.data}")
         raise InvalidRequest("SES-SNS callback failed: invalid JSON given", 400)
 
+    current_app.logger.info(f"Message type: {message_type}\nResponse data: {message}")
+
     try:
-        validatesns.validate(message, get_certificate=get_certificate)
-    except validatesns.ValidationError:
+        print("attempting to validate sns")
+        if valid_sns_message(message) == False:
+            current_app.logger.error(f"SES-SNS callback failed: validation failed! Response headers: {request.headers}\nResponse data: {request.data}\nError: Signature validation failed.")
+            print("attempting to validate sns failed")
+            raise InvalidRequest("SES-SNS callback failed: validation failed", 400)
+    except Exception as e:
+        current_app.logger.exception(f"SES-SNS callback failed: validation failed! Response headers: {request.headers}\nResponse data: {request.data}\nError: {e}")
         raise InvalidRequest("SES-SNS callback failed: validation failed", 400)
 
     if message.get('Type') == 'SubscriptionConfirmation':
-        url = message.get('SubscribeURL')
+        print("processing subscription")
+        url = message.get('SubscribeUrl') if 'SubscribeUrl' in message else message.get('SubscribeURL')
         response = requests.get(url)
         try:
             response.raise_for_status()
         except Exception as e:
-            current_app.logger.warning("Response: {}".format(response.text))
+            current_app.logger.warning(f"Attempt to raise_for_status()SubscriptionConfirmation Type message files for response: {response.text} with error {e}")
             raise e
 
         return jsonify(
             result="success", message="SES-SNS auto-confirm callback succeeded"
         ), 200
 
+    print("info logging")
+    # TODO remove after smoke testing on prod is implemented
+    current_app.logger.info(f"SNS message: {message} is a valid delivery status message. Attempting to process it now.")
+    
+    print("running process_ses_results")
     process_ses_results.apply_async([{"Message": message.get("Message")}], queue=QueueNames.NOTIFY)
 
+    print("returning success")
     return jsonify(
         result="success", message="SES-SNS callback succeeded"
     ), 200
@@ -101,10 +109,12 @@ def process_ses_results(self, response):
     try:
         ses_message = json.loads(response["Message"])
         notification_type = ses_message["notificationType"]
+        # TODO remove after smoke testing on prod is implemented
+        current_app.logger.info(f"Attempting to process SES delivery status message from SNS with type: {notification_type} and body: {ses_message}")
         bounce_message = None
 
         if notification_type == 'Bounce':
-            bounce_message = _determine_notification_bounce_type(ses_message)
+            bounce_message = determine_notification_bounce_type(ses_message)
         elif notification_type == 'Complaint':
             _check_and_queue_complaint_callback_task(*handle_complaint(ses_message))
             return True

app/celery/validate_sns.py

@@ -0,0 +1,113 @@
+import base64
+import re
+from urllib.parse import urlparse
+
+import requests
+from M2Crypto import X509
+
+from app import redis_store
+from app.config import Config
+
+USE_CACHE = True
+VALIDATE_ARN = True
+
+
+_signing_cert_cache = {}
+_cert_url_re = re.compile(
+    r'sns\.([a-z]{1,3}-[a-z]+-[0-9]{1,2})\.amazonaws\.com',
+)
+
+
+VALID_SNS_TOPICS = Config.VALID_SNS_TOPICS
+# VALID_SNS_TOPICS = ['my_bounce_topic_name', 'my_success_topic_name', 'my_complaint_topic_name']
+
+
+def get_certificate(url):
+    if USE_CACHE:
+        res = redis_store.get(url)
+        if res is not None:
+            return res
+        res = requests.get(url).text
+        redis_store.set(url, res, ex=60 * 60)  # 60 minutes
+        return res
+    else:
+        return requests.get(url).text
+
+
+def valid_sns_message(sns_payload):
+    """
+    Adapted from the solution posted at
+    https://github.com/boto/boto3/issues/2508#issuecomment-992931814
+    """
+    if not isinstance(sns_payload, dict):
+        return False
+
+    # Amazon SNS currently supports signature version 1.
+    if sns_payload.get('SignatureVersion') != '1':
+        return False
+    
+    if VALIDATE_ARN:
+        arn = sns_payload.get('TopicArn')
+        topic_name = arn.split(':')[5]
+        if topic_name not in VALID_SNS_TOPICS:
+            return False
+
+    payload_type = sns_payload.get('Type')
+    if payload_type in ['SubscriptionConfirmation', 'UnsubscribeConfirmation']:
+        fields = ['Message', 'MessageId', 'SubscribeURL', 'Timestamp', 'Token', 'TopicArn', 'Type']
+    elif payload_type == 'Notification':
+        fields = ['Message', 'MessageId', 'Subject', 'Timestamp', 'TopicArn', 'Type']
+    else:
+        return False
+
+    # Build the string to be signed.
+    string_to_sign = ''
+    for field in fields:
+        field_value = sns_payload.get(field)
+        if not isinstance(field_value, str):
+            return False
+        string_to_sign += field + '\n' + field_value + '\n'
+
+    # Get the signature
+    try:
+        decoded_signature = base64.b64decode(sns_payload.get('Signature'))
+    except (TypeError, ValueError):
+        return False
+
+    # Key signing cert url via Lambda and via webhook are slightly different
+    signing_cert_url = sns_payload.get('SigningCertUrl') if 'SigningCertUrl' in sns_payload else sns_payload.get('SigningCertURL')
+    if not isinstance(signing_cert_url, str):
+        return False
+    cert_scheme, cert_netloc, *_ = urlparse(signing_cert_url)
+    if cert_scheme != 'https' or not re.match(_cert_url_re, cert_netloc):
+        # The cert doesn't seem to be from AWS
+        return False
+    certificate = _signing_cert_cache.get(signing_cert_url)
+    if certificate is None:
+        certificate = X509.load_cert_string(get_certificate(signing_cert_url))
+        _signing_cert_cache[signing_cert_url] = certificate
+
+    if certificate.get_subject().as_text() != 'CN=sns.amazonaws.com':
+        return False
+
+    # Extract the public key.
+    public_key = certificate.get_pubkey()
+
+    # Amazon SNS uses SHA1withRSA.
+    # http://sns-public-resources.s3.amazonaws.com/SNS_Message_Signing_Release_Note_Jan_25_2011.pdf
+    public_key.reset_context(md='sha1')
+    public_key.verify_init()
+
+    # Sign the string.
+    public_key.verify_update(string_to_sign.encode())
+
+    # Verify the signature matches.
+    verification_result = public_key.verify_final(decoded_signature)
+
+    # M2Crypto uses EVP_VerifyFinal() from openssl as the underlying
+    # verification function. 1 indicates success, anything else is either
+    # a failure or an error.
+    if verification_result != 1:
+        return False
+
+    return True

app/config.py

@@ -119,6 +119,9 @@ class Config(object):
     
     # Use notify.sandbox.10x sending domain unless overwritten by environment
     NOTIFY_EMAIL_DOMAIN = 'notify.sandbox.10x.gsa.gov'
+    
+    # AWS SNS topics for delivery receipts
+    VALID_SNS_TOPICS = ['notify_test_bounce', 'notify_test_success', 'notify_test_complaint']
 
     # URL of redis instance
     REDIS_URL = os.environ.get('REDIS_URL')

app/notifications/callbacks.py

@@ -49,4 +49,4 @@ def create_complaint_callback_data(complaint, notification, service_callback_api
         "service_callback_api_bearer_token": service_callback_api.bearer_token,
     }
 
-    return encryption.encrypt(data)
+    return encryption.encrypt(data)

app/notifications/notifications_ses_callback.py

@@ -17,7 +17,7 @@ from app.models import Complaint
 from app.notifications.callbacks import create_complaint_callback_data
 
 
-def _determine_notification_bounce_type(ses_message):
+def determine_notification_bounce_type(ses_message):
     notification_type = ses_message["notificationType"]
     if notification_type in ["Delivery", "Complaint"]:
         return notification_type
@@ -51,7 +51,7 @@ def _determine_provider_response(ses_message):
 
 
 def get_aws_responses(ses_message):
-    status = _determine_notification_bounce_type(ses_message)
+    status = determine_notification_bounce_type(ses_message)
 
     base = {
         "Permanent": {

devcontainer-api/Dockerfile

@@ -21,6 +21,7 @@ RUN apt-get update \
     openssh-client \
     procps \
     sudo \
+    swig \
     tldr \
     unzip \
     vim \

requirements.in

@@ -16,10 +16,10 @@ itsdangerous==2.1.2
 jsonschema[format]==4.5.1
 marshmallow-sqlalchemy==0.28.1
 marshmallow==3.15.0
+M2Crypto==0.38.0
 psycopg2-binary==2.9.3
 PyJWT==2.4.0
 SQLAlchemy==1.4.40
-validatesns==0.1.1
 cachetools==5.1.0
 beautifulsoup4==4.11.1
 lxml==4.9.1

requirements.txt

@@ -10,8 +10,6 @@ amqp==5.1.1
     # via kombu
 arrow==1.2.2
     # via isoduration
-asn1crypto==1.5.1
-    # via oscrypto
 async-timeout==4.0.2
     # via redis
 attrs==21.4.0
@@ -150,6 +148,8 @@ kombu==5.2.4
     # via celery
 lxml==4.9.1
     # via -r requirements.in
+m2crypto==0.38.0
+    # via -r requirements.in
 mako==1.2.0
     # via alembic
 markupsafe==2.1.1
@@ -171,8 +171,6 @@ notifications-utils @ git+https://github.com/GSA/notifications-utils.git
     # via -r requirements.in
 orderedset==2.0.3
     # via notifications-utils
-oscrypto==1.3.0
-    # via validatesns
 packaging==21.3
     # via
     #   bleach
@@ -251,7 +249,6 @@ six==1.16.0
     #   flask-marshmallow
     #   python-dateutil
     #   rfc3339-validator
-    #   validatesns
 smartypants==2.0.1
     # via notifications-utils
 soupsieve==2.3.2.post1
@@ -272,8 +269,6 @@ urllib3==1.26.9
     # via
     #   botocore
     #   requests
-validatesns==0.1.1
-    # via -r requirements.in
 vine==5.0.0
     # via
     #   amqp

tests/app/celery/test_process_ses_receipts_tasks.py

@@ -71,7 +71,7 @@ def test_notifications_ses_400_with_certificate(client):
 
 
 def test_notifications_ses_200_autoconfirms_subscription(client, mocker):
-    mocker.patch("validatesns.validate")
+    mocker.patch("app.celery.process_ses_receipts_tasks.valid_sns_message", return_value=True)
     requests_mock = mocker.patch("requests.get")
     data = json.dumps({"Type": "SubscriptionConfirmation", "SubscribeURL": "https://foo"})
     response = client.post(
@@ -85,7 +85,7 @@ def test_notifications_ses_200_autoconfirms_subscription(client, mocker):
 
 
 def test_notifications_ses_200_call_process_task(client, mocker):
-    mocker.patch("validatesns.validate")
+    mocker.patch("app.celery.process_ses_receipts_tasks.valid_sns_message", return_value=True)
     process_mock = mocker.patch("app.celery.process_ses_receipts_tasks.process_ses_results.apply_async")
     data = {"Type": "Notification", "foo": "bar"}
     json_data = json.dumps(data)