test branch for notify-api-alt temporary deploy

Makefile

@@ -71,7 +71,7 @@ test: ## Run tests
 freeze-requirements: ## Pin all requirements including sub dependencies into requirements.txt
 	pip install --upgrade pip-tools
 	pip-compile requirements.in
-	pip3 install -r requirements.txt
+	pip3 install -r requirements.txt --no-cache-dir
 
 .PHONY: audit
 audit:

app/celery/process_ses_receipts_tasks.py

@@ -4,12 +4,14 @@ from json import decoder
 
 import iso8601
 import requests
+import traceback
 from celery.exceptions import Retry
 from flask import Blueprint, current_app, json, jsonify, request
 from sqlalchemy.orm.exc import NoResultFound
 
-from app import notify_celery, statsd_client
-from app.celery.validate_sns import valid_sns_message
+from app import notify_celery, statsd_client, redis_store
+# from app.celery.validate_sns import valid_sns_message
+import validatesns
 from app.config import QueueNames
 from app.dao import notifications_dao
 from app.errors import InvalidRequest, register_errors
@@ -23,6 +25,7 @@ from app.notifications.notifications_ses_callback import (
 )
 
 ses_callback_blueprint = Blueprint('notifications_ses_callback', __name__)
+DEFAULT_MAX_AGE = timedelta(days=10000)
 
 register_errors(ses_callback_blueprint)
 class SNSMessageType(enum.Enum):
@@ -41,6 +44,13 @@ def verify_message_type(message_type: str):
     except ValueError:
         raise InvalidMessageTypeException(f'{message_type} is not a valid message type.')
 
+def get_certificate(url):
+    res = redis_store.get(url)
+    if res is not None:
+        return res
+    res = requests.get(url).content
+    redis_store.set(url, res, ex=60 * 60)  # 60 minutes
+    return res
 
 # 400 counts as a permanent failure so SNS will not retry.
 # 500 counts as a failed delivery attempt so SNS will retry.
@@ -64,15 +74,26 @@ def sns_callback_handler():
         raise InvalidRequest("SES-SNS callback failed: invalid JSON given", 400)
 
     current_app.logger.info(f"Message type: {message_type}\nResponse data: {message}")
-
+    
     try:
-        if valid_sns_message(message) == False:
-            current_app.logger.error(f"SES-SNS callback failed: validation failed! Response headers: {request.headers}\nResponse data: {request.data}\nError: Signature validation failed.")
-            raise InvalidRequest("SES-SNS callback failed: validation failed", 400)
-    except Exception as e:
-        current_app.logger.exception(f"SES-SNS callback failed: validation failed! Response headers: {request.headers}\nResponse data: {request.data}\nError: {e}")
+        # AWS sends SigningCertURL if sending to a webhook, but SigningCertUrl if sending to a Lambda function
+        message["SigningCertURL"] = message["SigningCertURL"] if "SigningCertURL" in message else message["SigningCertUrl"] 
+        # Some SNS messages now contain "Subject": null, which is not handled by the validatesns library
+        if "Subject" in message and message["Subject"] == None:
+            message.pop("Subject")
+        validatesns.validate(message, get_certificate=get_certificate, max_age=DEFAULT_MAX_AGE)
+    except Exception as err:
+        current_app.logger.error(f"SES-SNS callback failed: validation failed! Response headers: {request.headers}\nResponse data: {request.data}\nError: Signature validation failed with error {err} and traceback {traceback.format_exc()}")
         raise InvalidRequest("SES-SNS callback failed: validation failed", 400)
 
+    # try:
+    #     if valid_sns_message(message) == False:
+    #         current_app.logger.error(f"SES-SNS callback failed: validation failed! Response headers: {request.headers}\nResponse data: {request.data}\nError: Signature validation failed.")
+    #         raise InvalidRequest("SES-SNS callback failed: validation failed", 400)
+    # except Exception as e:
+    #     current_app.logger.exception(f"SES-SNS callback failed: validation failed! Response headers: {request.headers}\nResponse data: {request.data}\nError: {e}")
+    #     raise InvalidRequest("SES-SNS callback failed: validation failed", 400)
+
     if message.get('Type') == 'SubscriptionConfirmation':
         url = message.get('SubscribeUrl') if 'SubscribeUrl' in message else message.get('SubscribeURL')
         response = requests.get(url)

app/cloudfoundry_config.py

@@ -6,7 +6,6 @@ def extract_cloudfoundry_config():
     vcap_services = json.loads(os.environ['VCAP_SERVICES'])
 
     # Postgres config
-    os.environ['SQLALCHEMY_DATABASE_URI'] = vcap_services['aws-rds'][0]['credentials']['uri'].replace('postgres',
-                                                                                                      'postgresql')
+    os.environ['SQLALCHEMY_DATABASE_URI'] = vcap_services['aws-rds'][0]['credentials']['uri'].replace('postgres','postgresql')
     # Redis config
-    os.environ['REDIS_URL'] = vcap_services['aws-elasticache-redis'][0]['credentials']['uri']
+    os.environ['REDIS_URL'] = vcap_services['aws-elasticache-redis'][0]['credentials']['uri'].replace('redis://','rediss://')

manifest.yml

@@ -1,7 +1,7 @@
 ---
 
 applications:
-  - name: notifications-api
+  - name: notify-api-alt
     buildpack: https://github.com/cloudfoundry/python-buildpack.git#v1.7.58
     instances: 1
     memory: 1G
@@ -9,22 +9,24 @@ applications:
     health-check-type: process
     health-check-invocation-timeout: 1
     routes:
-    - route: notifications-api.app.cloud.gov
+    - route: notify-api-alt.app.cloud.gov
 
     services:
-      - api-psql
-      - api-redis
+      - api-alt-psql
+      - api-alt-redis
 
     env:
+      BP_PIP_VERSION: latest
       NOTIFY_APP_NAME: api
       NOTIFY_LOG_PATH: /home/vcap/logs/app.log
       FLASK_APP: application.py
       FLASK_ENV: production
 
       NOTIFY_ENVIRONMENT: live
-      API_HOST_NAME: https://notifications-api.app.cloud.gov
-      ADMIN_BASE_URL: https://notifications-admin.app.cloud.gov
-      NOTIFICATION_QUEUE_PREFIX: prototype_10x
+      API_HOST_NAME: https://notify-api-alt.app.cloud.gov
+      ADMIN_BASE_URL: https://notify-admin-alt.app.cloud.gov
+      NOTIFICATION_QUEUE_PREFIX: notify_alt_
+      REDIS_ENABLED: true
       STATSD_HOST: localhost
 
       INTERNAL_CLIENT_API_KEYS: '{"notify-admin":["((ADMIN_CLIENT_SECRET))"]}'
@@ -33,6 +35,8 @@ applications:
       ADMIN_CLIENT_SECRET: ((ADMIN_CLIENT_SECRET))
       DANGEROUS_SALT: ((DANGEROUS_SALT))
       SECRET_KEY: ((SECRET_KEY))
+      AWS_ACCESS_KEY_ID: ((AWS_ACCESS_KEY_ID))
+      AWS_SECRET_ACCESS_KEY: ((AWS_SECRET_ACCESS_KEY))
       AWS_REGION: us-west-2
       AWS_PINPOINT_REGION: us-west-2
       AWS_US_TOLL_FREE_NUMBER: +18446120782

requirements.in

@@ -6,7 +6,7 @@ celery[sqs]==5.2.6
 Flask-Bcrypt==1.0.1
 flask-marshmallow==0.14.0
 Flask-Migrate==3.1.0
-git+https://github.com/pallets-eco/flask-sqlalchemy.git@aa7a61a5357cf6f5dcc135d98c781192457aa6fa#egg=Flask-SQLAlchemy==2.5.1
+Flask-SQLAlchemy==2.5.1
 Flask==2.1.2
 click-datetime==0.2
 # Should be pinned until a new gunicorn release greater than 20.1.0 comes out. (Due to eventlet v0.33 compatibility issues)
@@ -16,7 +16,6 @@ itsdangerous==2.1.2
 jsonschema[format]==4.5.1
 marshmallow-sqlalchemy==0.28.1
 marshmallow==3.15.0
-M2Crypto==0.38.0
 psycopg2-binary==2.9.3
 PyJWT==2.4.0
 SQLAlchemy==1.4.40
@@ -27,6 +26,8 @@ defusedxml==0.7.1
 Werkzeug==2.1.1
 python-dotenv==0.20.0
 
+validatesns==0.1.1
+
 notifications-python-client==6.3.0
 
 # PaaS

requirements.txt

@@ -10,6 +10,8 @@ amqp==5.1.1
     # via kombu
 arrow==1.2.2
     # via isoduration
+asn1crypto==1.5.1
+    # via oscrypto
 async-timeout==4.0.2
     # via redis
 attrs==21.4.0
@@ -99,7 +101,7 @@ flask-migrate==3.1.0
     # via -r requirements.in
 flask-redis==0.4.0
     # via notifications-utils
-flask-sqlalchemy @ git+https://github.com/pallets-eco/flask-sqlalchemy.git@aa7a61a5357cf6f5dcc135d98c781192457aa6fa
+flask-sqlalchemy==2.5.1
     # via
     #   -r requirements.in
     #   flask-migrate
@@ -148,8 +150,6 @@ kombu==5.2.4
     # via celery
 lxml==4.9.1
     # via -r requirements.in
-m2crypto==0.38.0
-    # via -r requirements.in
 mako==1.2.0
     # via alembic
 markupsafe==2.1.1
@@ -171,6 +171,8 @@ notifications-utils @ git+https://github.com/GSA/notifications-utils.git
     # via -r requirements.in
 orderedset==2.0.3
     # via notifications-utils
+oscrypto==1.3.0
+    # via validatesns
 packaging==21.3
     # via
     #   bleach
@@ -249,6 +251,7 @@ six==1.16.0
     #   flask-marshmallow
     #   python-dateutil
     #   rfc3339-validator
+    #   validatesns
 smartypants==2.0.1
     # via notifications-utils
 soupsieve==2.3.2.post1
@@ -269,6 +272,8 @@ urllib3==1.26.9
     # via
     #   botocore
     #   requests
+validatesns==0.1.1
+    # via -r requirements.in
 vine==5.0.0
     # via
     #   amqp