diff --git a/Makefile b/Makefile index 18caff76d..98fdef116 100644 --- a/Makefile +++ b/Makefile @@ -71,6 +71,7 @@ test: ## Run tests freeze-requirements: ## Pin all requirements including sub dependencies into requirements.txt pip install --upgrade pip-tools pip-compile requirements.in + pip3 install -r requirements.txt .PHONY: audit audit: diff --git a/app/celery/process_ses_receipts_tasks.py b/app/celery/process_ses_receipts_tasks.py index 8f743fa95..0e9a4dabf 100644 --- a/app/celery/process_ses_receipts_tasks.py +++ b/app/celery/process_ses_receipts_tasks.py @@ -4,21 +4,20 @@ from json import decoder import iso8601 import requests -import validatesns from celery.exceptions import Retry from flask import Blueprint, current_app, json, jsonify, request -from notifications_utils.statsd_decorators import statsd from sqlalchemy.orm.exc import NoResultFound -from app import notify_celery, redis_store, statsd_client +from app import notify_celery, statsd_client +from app.celery.validate_sns import valid_sns_message from app.config import QueueNames from app.dao import notifications_dao from app.errors import InvalidRequest, register_errors from app.models import NOTIFICATION_PENDING, NOTIFICATION_SENDING from app.notifications.notifications_ses_callback import ( _check_and_queue_complaint_callback_task, - _determine_notification_bounce_type, check_and_queue_callback_task, + determine_notification_bounce_type, get_aws_responses, handle_complaint, ) @@ -26,7 +25,6 @@ from app.notifications.notifications_ses_callback import ( ses_callback_blueprint = Blueprint('notifications_ses_callback', __name__) register_errors(ses_callback_blueprint) - class SNSMessageType(enum.Enum): SubscriptionConfirmation = 'SubscriptionConfirmation' Notification = 'Notification' @@ -44,14 +42,6 @@ def verify_message_type(message_type: str): raise InvalidMessageTypeException(f'{message_type} is not a valid message type.') -def get_certificate(url): - res = redis_store.get(url) - if res is not None: - return res - res = requests.get(url).content - redis_store.set(url, res, ex=60 * 60) # 60 minutes - return res - # 400 counts as a permanent failure so SNS will not retry. # 500 counts as a failed delivery attempt so SNS will retry. # See https://docs.aws.amazon.com/sns/latest/dg/DeliveryPolicies.html#DeliveryPolicies @@ -62,35 +52,53 @@ def get_certificate(url): def sns_callback_handler(): message_type = request.headers.get('x-amz-sns-message-type') try: + print("validating message type") verify_message_type(message_type) except InvalidMessageTypeException: + current_app.logger.exception(f"Response headers: {request.headers}\nResponse data: {request.data}") raise InvalidRequest("SES-SNS callback failed: invalid message type", 400) try: - message = json.loads(request.data) + print("loading message") + message = json.loads(request.data.decode('utf-8')) except decoder.JSONDecodeError: + current_app.logger.exception(f"Response headers: {request.headers}\nResponse data: {request.data}") raise InvalidRequest("SES-SNS callback failed: invalid JSON given", 400) + current_app.logger.info(f"Message type: {message_type}\nResponse data: {message}") + try: - validatesns.validate(message, get_certificate=get_certificate) - except validatesns.ValidationError: + print("attempting to validate sns") + if valid_sns_message(message) == False: + current_app.logger.error(f"SES-SNS callback failed: validation failed! Response headers: {request.headers}\nResponse data: {request.data}\nError: Signature validation failed.") + print("attempting to validate sns failed") + raise InvalidRequest("SES-SNS callback failed: validation failed", 400) + except Exception as e: + current_app.logger.exception(f"SES-SNS callback failed: validation failed! Response headers: {request.headers}\nResponse data: {request.data}\nError: {e}") raise InvalidRequest("SES-SNS callback failed: validation failed", 400) if message.get('Type') == 'SubscriptionConfirmation': - url = message.get('SubscribeURL') + print("processing subscription") + url = message.get('SubscribeUrl') if 'SubscribeUrl' in message else message.get('SubscribeURL') response = requests.get(url) try: response.raise_for_status() except Exception as e: - current_app.logger.warning("Response: {}".format(response.text)) + current_app.logger.warning(f"Attempt to raise_for_status()SubscriptionConfirmation Type message files for response: {response.text} with error {e}") raise e return jsonify( result="success", message="SES-SNS auto-confirm callback succeeded" ), 200 + print("info logging") + # TODO remove after smoke testing on prod is implemented + current_app.logger.info(f"SNS message: {message} is a valid delivery status message. Attempting to process it now.") + + print("running process_ses_results") process_ses_results.apply_async([{"Message": message.get("Message")}], queue=QueueNames.NOTIFY) + print("returning success") return jsonify( result="success", message="SES-SNS callback succeeded" ), 200 @@ -101,10 +109,12 @@ def process_ses_results(self, response): try: ses_message = json.loads(response["Message"]) notification_type = ses_message["notificationType"] + # TODO remove after smoke testing on prod is implemented + current_app.logger.info(f"Attempting to process SES delivery status message from SNS with type: {notification_type} and body: {ses_message}") bounce_message = None if notification_type == 'Bounce': - bounce_message = _determine_notification_bounce_type(ses_message) + bounce_message = determine_notification_bounce_type(ses_message) elif notification_type == 'Complaint': _check_and_queue_complaint_callback_task(*handle_complaint(ses_message)) return True diff --git a/app/celery/validate_sns.py b/app/celery/validate_sns.py new file mode 100644 index 000000000..e81e8fd15 --- /dev/null +++ b/app/celery/validate_sns.py @@ -0,0 +1,113 @@ +import base64 +import re +from urllib.parse import urlparse + +import requests +from M2Crypto import X509 + +from app import redis_store +from app.config import Config + +USE_CACHE = True +VALIDATE_ARN = True + + +_signing_cert_cache = {} +_cert_url_re = re.compile( + r'sns\.([a-z]{1,3}-[a-z]+-[0-9]{1,2})\.amazonaws\.com', +) + + +VALID_SNS_TOPICS = Config.VALID_SNS_TOPICS +# VALID_SNS_TOPICS = ['my_bounce_topic_name', 'my_success_topic_name', 'my_complaint_topic_name'] + + +def get_certificate(url): + if USE_CACHE: + res = redis_store.get(url) + if res is not None: + return res + res = requests.get(url).text + redis_store.set(url, res, ex=60 * 60) # 60 minutes + return res + else: + return requests.get(url).text + + +def valid_sns_message(sns_payload): + """ + Adapted from the solution posted at + https://github.com/boto/boto3/issues/2508#issuecomment-992931814 + """ + if not isinstance(sns_payload, dict): + return False + + # Amazon SNS currently supports signature version 1. + if sns_payload.get('SignatureVersion') != '1': + return False + + if VALIDATE_ARN: + arn = sns_payload.get('TopicArn') + topic_name = arn.split(':')[5] + if topic_name not in VALID_SNS_TOPICS: + return False + + payload_type = sns_payload.get('Type') + if payload_type in ['SubscriptionConfirmation', 'UnsubscribeConfirmation']: + fields = ['Message', 'MessageId', 'SubscribeURL', 'Timestamp', 'Token', 'TopicArn', 'Type'] + elif payload_type == 'Notification': + fields = ['Message', 'MessageId', 'Subject', 'Timestamp', 'TopicArn', 'Type'] + else: + return False + + # Build the string to be signed. + string_to_sign = '' + for field in fields: + field_value = sns_payload.get(field) + if not isinstance(field_value, str): + return False + string_to_sign += field + '\n' + field_value + '\n' + + # Get the signature + try: + decoded_signature = base64.b64decode(sns_payload.get('Signature')) + except (TypeError, ValueError): + return False + + # Key signing cert url via Lambda and via webhook are slightly different + signing_cert_url = sns_payload.get('SigningCertUrl') if 'SigningCertUrl' in sns_payload else sns_payload.get('SigningCertURL') + if not isinstance(signing_cert_url, str): + return False + cert_scheme, cert_netloc, *_ = urlparse(signing_cert_url) + if cert_scheme != 'https' or not re.match(_cert_url_re, cert_netloc): + # The cert doesn't seem to be from AWS + return False + certificate = _signing_cert_cache.get(signing_cert_url) + if certificate is None: + certificate = X509.load_cert_string(get_certificate(signing_cert_url)) + _signing_cert_cache[signing_cert_url] = certificate + + if certificate.get_subject().as_text() != 'CN=sns.amazonaws.com': + return False + + # Extract the public key. + public_key = certificate.get_pubkey() + + # Amazon SNS uses SHA1withRSA. + # http://sns-public-resources.s3.amazonaws.com/SNS_Message_Signing_Release_Note_Jan_25_2011.pdf + public_key.reset_context(md='sha1') + public_key.verify_init() + + # Sign the string. + public_key.verify_update(string_to_sign.encode()) + + # Verify the signature matches. + verification_result = public_key.verify_final(decoded_signature) + + # M2Crypto uses EVP_VerifyFinal() from openssl as the underlying + # verification function. 1 indicates success, anything else is either + # a failure or an error. + if verification_result != 1: + return False + + return True \ No newline at end of file diff --git a/app/config.py b/app/config.py index f1415e294..7aaf81ae9 100644 --- a/app/config.py +++ b/app/config.py @@ -119,6 +119,9 @@ class Config(object): # Use notify.sandbox.10x sending domain unless overwritten by environment NOTIFY_EMAIL_DOMAIN = 'notify.sandbox.10x.gsa.gov' + + # AWS SNS topics for delivery receipts + VALID_SNS_TOPICS = ['notify_test_bounce', 'notify_test_success', 'notify_test_complaint'] # URL of redis instance REDIS_URL = os.environ.get('REDIS_URL') diff --git a/app/notifications/callbacks.py b/app/notifications/callbacks.py index 7c3f1eed6..253e83af1 100644 --- a/app/notifications/callbacks.py +++ b/app/notifications/callbacks.py @@ -49,4 +49,4 @@ def create_complaint_callback_data(complaint, notification, service_callback_api "service_callback_api_bearer_token": service_callback_api.bearer_token, } - return encryption.encrypt(data) \ No newline at end of file + return encryption.encrypt(data) diff --git a/app/notifications/notifications_ses_callback.py b/app/notifications/notifications_ses_callback.py index a56de6f5a..472891418 100644 --- a/app/notifications/notifications_ses_callback.py +++ b/app/notifications/notifications_ses_callback.py @@ -17,7 +17,7 @@ from app.models import Complaint from app.notifications.callbacks import create_complaint_callback_data -def _determine_notification_bounce_type(ses_message): +def determine_notification_bounce_type(ses_message): notification_type = ses_message["notificationType"] if notification_type in ["Delivery", "Complaint"]: return notification_type @@ -51,7 +51,7 @@ def _determine_provider_response(ses_message): def get_aws_responses(ses_message): - status = _determine_notification_bounce_type(ses_message) + status = determine_notification_bounce_type(ses_message) base = { "Permanent": { diff --git a/devcontainer-api/Dockerfile b/devcontainer-api/Dockerfile index cdedfcc59..fd394946c 100644 --- a/devcontainer-api/Dockerfile +++ b/devcontainer-api/Dockerfile @@ -21,6 +21,7 @@ RUN apt-get update \ openssh-client \ procps \ sudo \ + swig \ tldr \ unzip \ vim \ diff --git a/requirements.in b/requirements.in index fec2b41c6..b82832472 100644 --- a/requirements.in +++ b/requirements.in @@ -16,10 +16,10 @@ itsdangerous==2.1.2 jsonschema[format]==4.5.1 marshmallow-sqlalchemy==0.28.1 marshmallow==3.15.0 +M2Crypto==0.38.0 psycopg2-binary==2.9.3 PyJWT==2.4.0 SQLAlchemy==1.4.40 -validatesns==0.1.1 cachetools==5.1.0 beautifulsoup4==4.11.1 lxml==4.9.1 diff --git a/requirements.txt b/requirements.txt index 2cdc474ae..ecbb8c22d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,8 +10,6 @@ amqp==5.1.1 # via kombu arrow==1.2.2 # via isoduration -asn1crypto==1.5.1 - # via oscrypto async-timeout==4.0.2 # via redis attrs==21.4.0 @@ -150,6 +148,8 @@ kombu==5.2.4 # via celery lxml==4.9.1 # via -r requirements.in +m2crypto==0.38.0 + # via -r requirements.in mako==1.2.0 # via alembic markupsafe==2.1.1 @@ -171,8 +171,6 @@ notifications-utils @ git+https://github.com/GSA/notifications-utils.git # via -r requirements.in orderedset==2.0.3 # via notifications-utils -oscrypto==1.3.0 - # via validatesns packaging==21.3 # via # bleach @@ -251,7 +249,6 @@ six==1.16.0 # flask-marshmallow # python-dateutil # rfc3339-validator - # validatesns smartypants==2.0.1 # via notifications-utils soupsieve==2.3.2.post1 @@ -272,8 +269,6 @@ urllib3==1.26.9 # via # botocore # requests -validatesns==0.1.1 - # via -r requirements.in vine==5.0.0 # via # amqp diff --git a/tests/app/celery/test_process_ses_receipts_tasks.py b/tests/app/celery/test_process_ses_receipts_tasks.py index 934f76c79..ad244c4f8 100644 --- a/tests/app/celery/test_process_ses_receipts_tasks.py +++ b/tests/app/celery/test_process_ses_receipts_tasks.py @@ -71,7 +71,7 @@ def test_notifications_ses_400_with_certificate(client): def test_notifications_ses_200_autoconfirms_subscription(client, mocker): - mocker.patch("validatesns.validate") + mocker.patch("app.celery.process_ses_receipts_tasks.valid_sns_message", return_value=True) requests_mock = mocker.patch("requests.get") data = json.dumps({"Type": "SubscriptionConfirmation", "SubscribeURL": "https://foo"}) response = client.post( @@ -85,7 +85,7 @@ def test_notifications_ses_200_autoconfirms_subscription(client, mocker): def test_notifications_ses_200_call_process_task(client, mocker): - mocker.patch("validatesns.validate") + mocker.patch("app.celery.process_ses_receipts_tasks.valid_sns_message", return_value=True) process_mock = mocker.patch("app.celery.process_ses_receipts_tasks.process_ses_results.apply_async") data = {"Type": "Notification", "foo": "bar"} json_data = json.dumps(data)