darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-09-10 18:23:01 -04:00
Code Issues Packages Projects Releases Wiki Activity

clean flake8 except provider code

Browse Source
This commit is contained in:
stvnrlly
2022-10-19 16:16:26 +00:00
parent 65f15b21b0
commit e9fdfd59f4
35 changed files with 178 additions and 166 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/authentication/auth.py
+4 -1
View File
@@ -18,7 +18,10 @@ from sqlalchemy.orm.exc import NoResultFound
from app.serialised_models import SerialisedService
GENERAL_TOKEN_ERROR_MESSAGE = 'Invalid token: make sure your API token matches the example at https://docs.notifications.service.gov.uk/rest-api.html#authorisation-header' # nosec B105
GENERAL_TOKEN_ERROR_MESSAGE = '''
Invalid token: make sure your API token matches the example
at https://docs.notifications.service.gov.uk/rest-api.html#authorisation-header
''' # nosec B105
AUTH_DB_CONNECTION_DURATION_SECONDS = Histogram(
'auth_db_connection_duration_seconds',
app/aws/s3.py
+10 -3
View File
@@ -10,18 +10,25 @@ default_access_key = os.environ.get('AWS_ACCESS_KEY_ID')
default_secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY')
default_region = os.environ.get('AWS_REGION')
def get_s3_file(bucket_name, file_location, access_key=default_access_key, secret_key=default_secret_key, region=default_region):
def get_s3_file(
bucket_name, file_location, access_key=default_access_key, secret_key=default_secret_key, region=default_region
):
s3_file = get_s3_object(bucket_name, file_location, access_key, secret_key, region)
return s3_file.get()['Body'].read().decode('utf-8')
def get_s3_object(bucket_name, file_location, access_key=default_access_key, secret_key=default_secret_key, region=default_region):
def get_s3_object(
bucket_name, file_location, access_key=default_access_key, secret_key=default_secret_key, region=default_region
):
session = Session(aws_access_key_id=access_key, aws_secret_access_key=secret_key, region_name=region)
s3 = session.resource('s3')
return s3.Object(bucket_name, file_location)
def file_exists(bucket_name, file_location, access_key=default_access_key, secret_key=default_secret_key, region=default_region):
def file_exists(
bucket_name, file_location, access_key=default_access_key, secret_key=default_secret_key, region=default_region
):
try:
# try and access metadata of object
get_s3_object(bucket_name, file_location, access_key, secret_key, region).metadata
app/celery/process_ses_receipts_tasks.py
+20 -8
View File
@@ -29,7 +29,10 @@ def process_ses_results(self, response):
ses_message = json.loads(response["Message"])
notification_type = ses_message["notificationType"]
# TODO remove after smoke testing on prod is implemented
current_app.logger.info(f"Attempting to process SES delivery status message from SNS with type: {notification_type} and body: {ses_message}")
current_app.logger.info(
f"Attempting to process SES delivery status message \
from SNS with type: {notification_type} and body: {ses_message}"
)
bounce_message = None
if notification_type == 'Bounce':
@@ -49,13 +52,16 @@ def process_ses_results(self, response):
message_time = iso8601.parse_date(ses_message["mail"]["timestamp"]).replace(tzinfo=None)
if datetime.utcnow() - message_time < timedelta(minutes=5):
current_app.logger.info(
f"notification not found for reference: {reference} (while attempting update to {notification_status}). "
f"Callback may have arrived before notification was persisted to the DB. Adding task to retry queue"
f"notification not found for reference: {reference} \
(while attempting update to {notification_status}). "
f"Callback may have arrived before notification was \
persisted to the DB. Adding task to retry queue"
)
self.retry(queue=QueueNames.RETRY)
else:
current_app.logger.warning(
"notification not found for reference: {} (while attempting update to {})".format(reference, notification_status)
"notification not found for reference: {} (while \
attempting update to {})".format(reference, notification_status)
)
return
@@ -102,6 +108,7 @@ def process_ses_results(self, response):
current_app.logger.exception("Error processing SES results: {}".format(type(e)))
self.retry(queue=QueueNames.RETRY)
def determine_notification_bounce_type(ses_message):
notification_type = ses_message["notificationType"]
if notification_type in ["Delivery", "Complaint"]:
@@ -116,14 +123,16 @@ def determine_notification_bounce_type(ses_message):
return "Permanent"
return "Temporary"
def determine_notification_type(ses_message):
notification_type = ses_message["notificationType"]
if notification_type not in ["Bounce","Complaint","Delivery"]:
if notification_type not in ["Bounce", "Complaint", "Delivery"]:
raise KeyError(f"Unhandled sns notification type {notification_type}")
if notification_type == 'Bounce':
return determine_notification_bounce_type(ses_message)
return notification_type
def _determine_provider_response(ses_message):
if ses_message["notificationType"] != "Bounce":
return None
@@ -175,7 +184,9 @@ def get_aws_responses(ses_message):
def handle_complaint(ses_message):
recipient_email = remove_emails_from_complaint(ses_message)[0]
current_app.logger.info("Complaint from SES: \n{}".format(json.dumps(ses_message).replace("{", "(").replace("}", ")")))
current_app.logger.info(
"Complaint from SES: \n{}".format(json.dumps(ses_message).replace("{", "(").replace("}", ")"))
)
try:
reference = ses_message["mail"]["messageId"]
except KeyError as e:
@@ -219,7 +230,9 @@ def check_and_queue_callback_task(notification):
service_callback_api = get_service_delivery_status_callback_api_for_service(service_id=notification.service_id)
if service_callback_api:
notification_data = create_delivery_status_callback_data(notification, service_callback_api)
send_delivery_status_to_service.apply_async([str(notification.id), notification_data], queue=QueueNames.CALLBACKS)
send_delivery_status_to_service.apply_async(
[str(notification.id), notification_data], queue=QueueNames.CALLBACKS
)
def _check_and_queue_complaint_callback_task(complaint, notification, recipient):
@@ -228,4 +241,3 @@ def _check_and_queue_complaint_callback_task(complaint, notification, recipient)
if service_callback_api:
complaint_data = create_complaint_callback_data(complaint, notification, service_callback_api, recipient)
send_complaint_to_service.apply_async([complaint_data], queue=QueueNames.CALLBACKS)
app/celery/process_sms_client_response_tasks.py
-1
View File
@@ -1,7 +1,6 @@
import uuid
from datetime import datetime
import pytest
from flask import current_app
from notifications_utils.template import SMSMessageTemplate
app/celery/research_mode_tasks.py
+1 -1
View File
@@ -124,7 +124,7 @@ def create_fake_letter_response_file(self, reference):
dvla_response_data = '{}|Sent|0|Sorted'.format(reference)
# try and find a filename that hasn't been taken yet - from a random time within the last 30 seconds
for i in sorted(range(30), key=lambda _: random.random()): # nosec B311 - not security related
for i in sorted(range(30), key=lambda _: random.random()): # nosec B311 - not security related
upload_file_name = 'NOTIFY-{}-RSP.TXT'.format((now - timedelta(seconds=i)).strftime('%Y%m%d%H%M%S'))
if not file_exists(current_app.config['DVLA_RESPONSE_BUCKET_NAME'], upload_file_name):
break
app/celery/service_callback_tasks.py
+1 -1
View File
@@ -114,7 +114,7 @@ def create_delivery_status_callback_data(notification, service_callback_api):
"notification_client_reference": notification.client_reference,
"notification_to": notification.to,
"notification_status": notification.status,
"notification_provider_response": notification.provider_response, # TODO do we have a test for provider_response
"notification_provider_response": notification.provider_response, # TODO do we test for provider_response?
"notification_created_at": notification.created_at.strftime(DATETIME_FORMAT),
"notification_updated_at":
notification.updated_at.strftime(DATETIME_FORMAT) if notification.updated_at else None,
app/clients/__init__.py
+1 -4
View File
@@ -1,6 +1,3 @@
from celery import current_app
class ClientException(Exception):
'''
Base Exceptions for sending notifications that fail
@@ -38,7 +35,7 @@ class NotificationProviderClients(object):
return self.email_clients.get(name)
def get_client_by_name_and_type(self, name, notification_type):
assert notification_type in ['email', 'sms'] # nosec B101
assert notification_type in ['email', 'sms'] # nosec B101
if notification_type == 'email':
return self.get_email_client(name)
app/cloudfoundry_config.py
+9 -4
View File
@@ -8,16 +8,20 @@ def find_by_service_name(services, service_name):
return services[i]
return None
def extract_cloudfoundry_config():
vcap_services = json.loads(os.environ['VCAP_SERVICES'])
# Postgres config
os.environ['SQLALCHEMY_DATABASE_URI'] = vcap_services['aws-rds'][0]['credentials']['uri'].replace('postgres','postgresql')
os.environ['SQLALCHEMY_DATABASE_URI'] = \
vcap_services['aws-rds'][0]['credentials']['uri'].replace('postgres', 'postgresql')
# Redis config
os.environ['REDIS_URL'] = vcap_services['aws-elasticache-redis'][0]['credentials']['uri'].replace('redis://','rediss://')
os.environ['REDIS_URL'] = \
vcap_services['aws-elasticache-redis'][0]['credentials']['uri'].replace('redis://', 'rediss://')
# CSV Upload Bucket Name
bucket_service = find_by_service_name(vcap_services['s3'], f"notifications-api-csv-upload-bucket-{os.environ['DEPLOY_ENV']}")
bucket_service = \
find_by_service_name(vcap_services['s3'], f"notifications-api-csv-upload-bucket-{os.environ['DEPLOY_ENV']}")
if bucket_service:
os.environ['CSV_UPLOAD_BUCKET_NAME'] = bucket_service['credentials']['bucket']
os.environ['CSV_UPLOAD_ACCESS_KEY'] = bucket_service['credentials']['access_key_id']
@@ -25,7 +29,8 @@ def extract_cloudfoundry_config():
os.environ['CSV_UPLOAD_REGION'] = bucket_service['credentials']['region']
# Contact List Bucket Name
bucket_service = find_by_service_name(vcap_services['s3'], f"notifications-api-contact-list-bucket-{os.environ['DEPLOY_ENV']}")
bucket_service = \
find_by_service_name(vcap_services['s3'], f"notifications-api-contact-list-bucket-{os.environ['DEPLOY_ENV']}")
if bucket_service:
os.environ['CONTACT_LIST_BUCKET_NAME'] = bucket_service['credentials']['bucket']
os.environ['CONTACT_LIST_ACCESS_KEY'] = bucket_service['credentials']['access_key_id']
app/commands.py
-80
View File
@@ -142,86 +142,6 @@ def purge_functional_test_data(user_email_prefix):
delete_model_user(usr)
@notify_command()
def backfill_notification_statuses():
"""
DEPRECATED. Populates notification_status.
This will be used to populate the new `Notification._status_fkey` with the old
`Notification._status_enum`
"""
LIMIT = 250000
subq = "SELECT id FROM notification_history WHERE notification_status is NULL LIMIT {}".format(LIMIT) # nosec B608 no user-controlled input
update = "UPDATE notification_history SET notification_status = status WHERE id in ({})".format(subq) # nosec B608 no user-controlled input
result = db.session.execute(subq).fetchall()
while len(result) > 0:
db.session.execute(update)
print('commit {} updates at {}'.format(LIMIT, datetime.utcnow()))
db.session.commit()
result = db.session.execute(subq).fetchall()
@notify_command()
def update_notification_international_flag():
"""
DEPRECATED. Set notifications.international=false.
"""
# 250,000 rows takes 30 seconds to update.
subq = "select id from notifications where international is null limit 250000"
update = "update notifications set international = False where id in ({})".format(subq) # nosec B608 no user-controlled input
result = db.session.execute(subq).fetchall()
while len(result) > 0:
db.session.execute(update)
print('commit 250000 updates at {}'.format(datetime.utcnow()))
db.session.commit()
result = db.session.execute(subq).fetchall()
# Now update notification_history
subq_history = "select id from notification_history where international is null limit 250000"
update_history = "update notification_history set international = False where id in ({})".format(subq_history) # nosec B608 no user-controlled input
result_history = db.session.execute(subq_history).fetchall()
while len(result_history) > 0:
db.session.execute(update_history)
print('commit 250000 updates at {}'.format(datetime.utcnow()))
db.session.commit()
result_history = db.session.execute(subq_history).fetchall()
@notify_command()
def fix_notification_statuses_not_in_sync():
"""
DEPRECATED.
This will be used to correct an issue where Notification._status_enum and NotificationHistory._status_fkey
became out of sync. See 979e90a.
Notification._status_enum is the source of truth so NotificationHistory._status_fkey will be updated with
these values.
"""
MAX = 10000
subq = "SELECT id FROM notifications WHERE cast (status as text) != notification_status LIMIT {}".format(MAX) # nosec B608 no user-controlled input
update = "UPDATE notifications SET notification_status = status WHERE id in ({})".format(subq) # nosec B608 no user-controlled input
result = db.session.execute(subq).fetchall()
while len(result) > 0:
db.session.execute(update)
print('Committed {} updates at {}'.format(len(result), datetime.utcnow()))
db.session.commit()
result = db.session.execute(subq).fetchall()
subq_hist = "SELECT id FROM notification_history WHERE cast (status as text) != notification_status LIMIT {}".format(MAX) # nosec B608
update = "UPDATE notification_history SET notification_status = status WHERE id in ({})".format(subq_hist) # nosec B608 no user-controlled input
result = db.session.execute(subq_hist).fetchall()
while len(result) > 0:
db.session.execute(update)
print('Committed {} updates at {}'.format(len(result), datetime.utcnow()))
db.session.commit()
result = db.session.execute(subq_hist).fetchall()
@notify_command(name='insert-inbound-numbers')
@click.option('-f', '--file_name', required=True,
help="""Full path of the file to upload, file is a contains inbound numbers,
app/config.py
+21 -9
View File
@@ -89,11 +89,11 @@ class Config(object):
# secrets that internal apps, such as the admin app or document download, must use to authenticate with the API
ADMIN_CLIENT_ID = 'notify-admin'
GOVUK_ALERTS_CLIENT_ID = 'govuk-alerts' # TODO: can remove?
GOVUK_ALERTS_CLIENT_ID = 'govuk-alerts' # TODO: can remove?
INTERNAL_CLIENT_API_KEYS = json.loads(
os.environ.get('INTERNAL_CLIENT_API_KEYS', '{"notify-admin":["dev-notify-secret-key"]}')
) # TODO: handled by varsfile?
) # TODO: handled by varsfile?
# encyption secret/salt
ADMIN_CLIENT_SECRET = os.environ.get('ADMIN_CLIENT_SECRET')
@@ -165,7 +165,7 @@ class Config(object):
MAX_VERIFY_CODE_COUNT = 5
MAX_FAILED_LOGIN_COUNT = 10
SES_STUB_URL = None # TODO: set to a URL in env and remove this to use a stubbed SES service
SES_STUB_URL = None # TODO: set to a URL in env and remove this to use a stubbed SES service
# be careful increasing this size without being sure that we won't see slowness in pysftp
MAX_LETTER_PDF_ZIP_FILESIZE = 40 * 1024 * 1024 # 40mb
@@ -186,7 +186,7 @@ class Config(object):
SMS_CODE_TEMPLATE_ID = '36fb0730-6259-4da1-8a80-c8de22ad4246'
EMAIL_2FA_TEMPLATE_ID = '299726d2-dba6-42b8-8209-30e1d66ea164'
NEW_USER_EMAIL_VERIFICATION_TEMPLATE_ID = 'ece42649-22a8-4d06-b87f-d52d5d3f0a27'
PASSWORD_RESET_TEMPLATE_ID = '474e9242-823b-4f99-813d-ed392e7f1201' # nosec B105 - this is not a password
PASSWORD_RESET_TEMPLATE_ID = '474e9242-823b-4f99-813d-ed392e7f1201' # nosec B105 - this is not a password
ALREADY_REGISTERED_EMAIL_TEMPLATE_ID = '0880fbb1-a0c6-46f0-9a8e-36c986381ceb'
CHANGE_EMAIL_CONFIRMATION_TEMPLATE_ID = 'eb4d9930-87ab-4aef-9bce-786762687884'
SERVICE_NOW_LIVE_TEMPLATE_ID = '618185c6-3636-49cd-b7d2-6f6f5eb3bdde'
@@ -437,7 +437,7 @@ class Development(Config):
# Config.GOVUK_ALERTS_CLIENT_ID: ['govuk-alerts-secret-key']
# }
SECRET_KEY = 'dev-notify-secret-key' # nosec B105 - this is only used in development
SECRET_KEY = 'dev-notify-secret-key' # nosec B105 - this is only used in development
DANGEROUS_SALT = 'dev-notify-salt'
MMG_INBOUND_SMS_AUTH = ['testkey']
@@ -448,7 +448,10 @@ class Development(Config):
NOTIFY_EMAIL_DOMAIN = os.getenv('NOTIFY_EMAIL_DOMAIN', 'notify.sandbox.10x.gsa.gov')
SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI', 'postgresql://postgres:chummy@db:5432/notification_api')
SQLALCHEMY_DATABASE_URI = os.environ.get(
'SQLALCHEMY_DATABASE_URI',
'postgresql://postgres:chummy@db:5432/notification_api'
)
ANTIVIRUS_ENABLED = os.environ.get('ANTIVIRUS_ENABLED') == '1'
@@ -486,7 +489,10 @@ class Test(Development):
# LETTER_SANITISE_BUCKET_NAME = 'test-letters-sanitise'
# this is overriden in CI
SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_TEST_URI', 'postgresql://postgres:chummy@db:5432/test_notification_api')
SQLALCHEMY_DATABASE_URI = os.getenv(
'SQLALCHEMY_DATABASE_TEST_URI',
'postgresql://postgres:chummy@db:5432/test_notification_api'
)
CELERY = {
**Config.CELERY,
@@ -546,11 +552,17 @@ class Staging(Config):
class Live(Config):
NOTIFY_ENVIRONMENT = 'live'
# buckets
CSV_UPLOAD_BUCKET_NAME = os.environ.get('CSV_UPLOAD_BUCKET_NAME', 'notifications-prototype-csv-upload') # created in gsa sandbox
CSV_UPLOAD_BUCKET_NAME = os.environ.get(
'CSV_UPLOAD_BUCKET_NAME',
'notifications-prototype-csv-upload'
) # created in gsa sandbox
CSV_UPLOAD_ACCESS_KEY = os.environ.get('CSV_UPLOAD_ACCESS_KEY')
CSV_UPLOAD_SECRET_KEY = os.environ.get('CSV_UPLOAD_SECRET_KEY')
CSV_UPLOAD_REGION = os.environ.get('CSV_UPLOAD_REGION')
CONTACT_LIST_BUCKET_NAME = os.environ.get('CONTACT_LIST_BUCKET_NAME', 'notifications-prototype-contact-list-upload') # created in gsa sandbox
CONTACT_LIST_BUCKET_NAME = os.environ.get(
'CONTACT_LIST_BUCKET_NAME',
'notifications-prototype-contact-list-upload'
) # created in gsa sandbox
CONTACT_LIST_ACCESS_KEY = os.environ.get('CONTACT_LIST_ACCESS_KEY')
CONTACT_LIST_SECRET_KEY = os.environ.get('CONTACT_LIST_SECRET_KEY')
CONTACT_LIST_REGION = os.environ.get('CONTACT_LIST_REGION')
app/dao/notifications_dao.py
+2
View File
@@ -87,6 +87,7 @@ def country_records_delivery(phone_prefix):
dlr = INTERNATIONAL_BILLING_RATES[phone_prefix]['attributes']['dlr']
return dlr and dlr.lower() == 'yes'
def _decide_permanent_temporary_failure(current_status, status):
# If we go from pending to delivered we need to set failure type as temporary-failure
if current_status == NOTIFICATION_PENDING and status == NOTIFICATION_PERMANENT_FAILURE:
@@ -102,6 +103,7 @@ def _update_notification_status(notification, status, provider_response=None):
dao_update_notification(notification)
return notification
@autocommit
def update_notification_status_by_id(notification_id, status, sent_by=None, detailed_status_code=None):
notification = Notification.query.with_for_update().filter(Notification.id == notification_id).first()
app/delivery/send_to_providers.py
+5 -3
View File
@@ -175,10 +175,12 @@ provider_cache = TTLCache(maxsize=8, ttl=10)
@cached(cache=provider_cache)
def provider_to_use(notification_type, international=True):
international = False # TODO: remove or resolve the functionality of this flag
international = False # TODO: remove or resolve the functionality of this flag
# TODO rip firetext and mmg out of early migrations and clean up the expression below
active_providers = [
p for p in get_provider_details_by_notification_type(notification_type, international) if p.active and p.identifier not in ['firetext','mmg']
p for p in get_provider_details_by_notification_type(
notification_type, international
) if p.active and p.identifier not in ['firetext', 'mmg']
]
if not active_providers:
@@ -191,7 +193,7 @@ def provider_to_use(notification_type, international=True):
chosen_provider = active_providers[0]
else:
weights = [p.priority for p in active_providers]
chosen_provider = random.choices(active_providers, weights=weights)[0] # nosec B311 - this is not security/cryptography related
chosen_provider = random.choices(active_providers, weights=weights)[0] # nosec B311 - not sec/crypto related
return notification_provider_clients.get_client_by_name_and_type(chosen_provider.identifier, notification_type)
app/inbound_sms/rest.py
+1 -1
View File
@@ -1,5 +1,5 @@
from flask import Blueprint, jsonify, request
from notifications_utils.recipients import try_validate_and_format_phone_number
# from notifications_utils.recipients import try_validate_and_format_phone_number
from app.dao.inbound_sms_dao import (
dao_count_inbound_sms_for_service,
app/models.py
+5 -3
View File
@@ -122,7 +122,9 @@ class User(db.Model):
state = db.Column(db.String, nullable=False, default='pending')
platform_admin = db.Column(db.Boolean, nullable=False, default=False)
current_session_id = db.Column(UUID(as_uuid=True), nullable=True)
auth_type = db.Column(db.String, db.ForeignKey('auth_type.name'), index=True, nullable=False, default=EMAIL_AUTH_TYPE)
auth_type = db.Column(
db.String, db.ForeignKey('auth_type.name'), index=True, nullable=False, default=EMAIL_AUTH_TYPE
)
email_access_validated_at = db.Column(
db.DateTime, index=False, unique=False, nullable=False, default=datetime.datetime.utcnow
)
@@ -608,7 +610,7 @@ class AnnualBilling(db.Model):
"name": self.service.name
}
return{
return {
"id": str(self.id),
'free_sms_fragment_limit': self.free_sms_fragment_limit,
'service_id': self.service_id,
@@ -1645,7 +1647,7 @@ class Notification(db.Model):
"""
# this should only ever be called for letter notifications - it makes no sense otherwise and I'd rather not
# get the two code flows mixed up at all
assert self.notification_type == LETTER_TYPE # nosec B101 - current calling code already validates the correct type
assert self.notification_type == LETTER_TYPE # nosec B101 - current calling code validates correct type
if self.status in [NOTIFICATION_CREATED, NOTIFICATION_SENDING]:
return NOTIFICATION_STATUS_LETTER_ACCEPTED
app/notifications/notifications_ses_callback.py
+1
View File
@@ -10,6 +10,7 @@ from app.notifications.sns_handlers import sns_notification_handler
ses_callback_blueprint = Blueprint('notifications_ses_callback', __name__)
DEFAULT_MAX_AGE = timedelta(days=10000)
# 400 counts as a permanent failure so SNS will not retry.
# 500 counts as a failed delivery attempt so SNS will retry.
# See https://docs.aws.amazon.com/sns/latest/dg/DeliveryPolicies.html#DeliveryPolicies
app/notifications/notifications_sms_callback.py
+3 -3
View File
@@ -1,10 +1,10 @@
from flask import Blueprint, json, jsonify, request
from flask import Blueprint # , json, jsonify, request
# from app.celery.process_sms_client_response_tasks import (
# process_sms_client_response,
# )
from app.config import QueueNames
from app.errors import InvalidRequest, register_errors
# from app.config import QueueNames
from app.errors import register_errors
sms_callback_blueprint = Blueprint("sms_callback", __name__, url_prefix="/notifications/sms")
register_errors(sms_callback_blueprint)
app/notifications/receive_notifications.py
+6 -3
View File
@@ -24,6 +24,7 @@ INBOUND_SMS_COUNTER = Counter(
['provider']
)
@receive_notifications_blueprint.route('/notifications/sms/receive/sns', methods=['POST'])
def receive_sns_sms():
"""
@@ -53,13 +54,16 @@ def receive_sns_sms():
# TODO wrap this up
if "inboundMessageId" in message:
# TODO use standard formatting we use for all US numbers
inbound_number = message['destinationNumber'].replace('+','')
inbound_number = message['destinationNumber'].replace('+', '')
service = fetch_potential_service(inbound_number, 'sns')
if not service:
# since this is an issue with our service <-> number mapping, or no inbound_sms service permission
# we should still tell SNS that we received it successfully
current_app.logger.warning(f"Mapping between service and inbound number: {inbound_number} is broken, or service does not have permission to receive inbound sms")
current_app.logger.warning(
f"Mapping between service and inbound number: {inbound_number} is broken, \
or service does not have permission to receive inbound sms"
)
return jsonify(
result="success", message="SMS-SNS callback succeeded"
), 200
@@ -79,7 +83,6 @@ def receive_sns_sms():
date_received=date_received,
provider_name=provider_name)
# TODO ensure inbound sms callback endpoints are accessible and functioning for notify api users, then uncomment the task below
tasks.send_inbound_sms_to_service.apply_async([str(inbound.id), str(service.id)], queue=QueueNames.NOTIFY)
current_app.logger.debug(
app/notifications/sns_cert_validator.py
+4 -2
View File
@@ -19,6 +19,7 @@ _cert_url_re = re.compile(
r'sns\.([a-z]{1,3}-[a-z]+-[0-9]{1,2})\.amazonaws\.com',
)
class ValidationError(Exception):
"""
ValidationError. Raised when a message fails integrity checks.
@@ -56,7 +57,7 @@ def get_string_to_sign(sns_payload):
for field in fields:
field_value = sns_payload.get(field)
if not isinstance(field_value, str):
if field == 'Subject' and field_value == None:
if field == 'Subject' and field_value is None:
continue
raise ValidationError(f"In {field}, found non-string value: {field_value}")
string_to_sign += field + '\n' + field_value + '\n'
@@ -83,7 +84,8 @@ def validate_sns_cert(sns_payload):
string_to_sign = get_string_to_sign(sns_payload)
# Key signing cert url via Lambda and via webhook are slightly different
signing_cert_url = sns_payload.get('SigningCertUrl') if 'SigningCertUrl' in sns_payload else sns_payload.get('SigningCertURL')
signing_cert_url = sns_payload.get('SigningCertUrl') if 'SigningCertUrl' in \
sns_payload else sns_payload.get('SigningCertURL')
if not isinstance(signing_cert_url, str):
raise ValidationError("Signing cert url must be a string")
cert_scheme, cert_netloc, *_ = urlparse(signing_cert_url)
app/notifications/sns_handlers.py
+11 -3
View File
@@ -45,7 +45,9 @@ def sns_notification_handler(data, headers):
try:
validate_sns_cert(message)
except Exception as e:
current_app.logger.error(f"SES-SNS callback failed: validation failed with error: Signature validation failed with error {e}")
current_app.logger.error(
f"SES-SNS callback failed: validation failed with error: Signature validation failed with error {e}"
)
raise InvalidRequest("SES-SNS callback failed: validation failed", 400)
if message.get('Type') == 'SubscriptionConfirmation':
@@ -55,8 +57,14 @@ def sns_notification_handler(data, headers):
try:
response.raise_for_status()
except Exception as e:
current_app.logger.warning(f"Attempt to raise_for_status()SubscriptionConfirmation Type message files for response: {response.text} with error {e}")
raise InvalidRequest("SES-SNS callback failed: attempt to raise_for_status()SubscriptionConfirmation Type message failed", 400)
current_app.logger.warning(
f"Attempt to raise_for_status()SubscriptionConfirmation Type \
message files for response: {response.text} with error {e}"
)
raise InvalidRequest(
"SES-SNS callback failed: attempt to raise_for_status()SubscriptionConfirmation \
Type message failed", 400
)
current_app.logger.info("SES-SNS auto-confirm subscription callback succeeded")
return message
tests/app/celery/test_process_ses_receipts_tasks.py
+22 -4
View File
@@ -87,7 +87,7 @@ def test_notifications_ses_200_autoconfirms_subscription(client, mocker):
def test_notifications_ses_200_call_process_task(client, mocker):
process_mock = mocker.patch("app.notifications.notifications_ses_callback.process_ses_results.apply_async")
mocker.patch("app.notifications.sns_handlers.validate_sns_cert", return_value=True)
data = {"Type": "Notification", "foo": "bar", "Message": {"mail": "baz"} }
data = {"Type": "Notification", "foo": "bar", "Message": {"mail": "baz"}}
mocker.patch("app.notifications.sns_handlers.sns_notification_handler", return_value=data)
json_data = json.dumps(data)
response = client.post(
@@ -156,7 +156,10 @@ def test_ses_callback_should_update_notification_status(
status='sending',
sent_at=datetime.utcnow()
)
callback_api = create_service_callback_api(service=sample_email_template.service, url="https://original_url.com")
callback_api = create_service_callback_api(
service=sample_email_template.service,
url="https://original_url.com"
)
assert get_notification_by_id(notification.id).status == 'sending'
assert process_ses_results(ses_notification_callback(reference='ref'))
assert get_notification_by_id(notification.id).status == 'delivered'
@@ -186,13 +189,19 @@ def test_ses_callback_should_retry_if_notification_is_new(mocker):
assert process_ses_results(ses_notification_callback(reference='ref')) is None
assert mock_logger.call_count == 0
assert mock_retry.call_count == 1
def test_ses_callback_should_log_if_notification_is_missing(client, _notify_db, mocker):
mock_retry = mocker.patch('app.celery.process_ses_receipts_tasks.process_ses_results.retry')
mock_logger = mocker.patch('app.celery.process_ses_receipts_tasks.current_app.logger.warning')
with freeze_time('2017-11-17T12:34:03.646Z'):
assert process_ses_results(ses_notification_callback(reference='ref')) is None
assert mock_retry.call_count == 0
mock_logger.assert_called_once_with('notification not found for reference: ref (while attempting update to delivered)')
mock_logger.assert_called_once_with(
'notification not found for reference: ref (while attempting update to delivered)'
)
def test_ses_callback_should_not_retry_if_notification_is_old(mocker):
mock_retry = mocker.patch('app.celery.process_ses_receipts_tasks.process_ses_results.retry')
mock_logger = mocker.patch('app.celery.process_ses_receipts_tasks.current_app.logger.error')
@@ -200,6 +209,8 @@ def test_ses_callback_should_not_retry_if_notification_is_old(mocker):
assert process_ses_results(ses_notification_callback(reference='ref')) is None
assert mock_logger.call_count == 0
assert mock_retry.call_count == 0
def test_ses_callback_does_not_call_send_delivery_status_if_no_db_entry(
client,
_notify_db,
@@ -222,6 +233,8 @@ def test_ses_callback_does_not_call_send_delivery_status_if_no_db_entry(
assert process_ses_results(ses_notification_callback(reference='ref'))
assert get_notification_by_id(notification.id).status == 'delivered'
send_mock.assert_not_called()
def test_ses_callback_should_update_multiple_notification_status_sent(
client,
_notify_db,
@@ -257,6 +270,8 @@ def test_ses_callback_should_update_multiple_notification_status_sent(
assert process_ses_results(ses_notification_callback(reference='ref2'))
assert process_ses_results(ses_notification_callback(reference='ref3'))
assert send_mock.called
def test_ses_callback_should_set_status_to_temporary_failure(client,
_notify_db,
notify_db_session,
@@ -278,6 +293,8 @@ def test_ses_callback_should_set_status_to_temporary_failure(client,
assert process_ses_results(ses_soft_bounce_callback(reference='ref'))
assert get_notification_by_id(notification.id).status == 'temporary-failure'
assert send_mock.called
def test_ses_callback_should_set_status_to_permanent_failure(client,
_notify_db,
notify_db_session,
@@ -299,6 +316,8 @@ def test_ses_callback_should_set_status_to_permanent_failure(client,
assert process_ses_results(ses_hard_bounce_callback(reference='ref'))
assert get_notification_by_id(notification.id).status == 'permanent-failure'
assert send_mock.called
def test_ses_callback_should_send_on_complaint_to_user_callback_api(sample_email_template, mocker):
send_mock = mocker.patch(
'app.celery.service_callback_tasks.send_complaint_to_service.apply_async'
@@ -321,4 +340,3 @@ def test_ses_callback_should_send_on_complaint_to_user_callback_api(sample_email
'service_callback_api_url': 'https://original_url.com',
'to': 'recipient1@example.com'
}
tests/app/celery/test_process_sms_client_response_tasks.py
+3 -3
View File
@@ -5,9 +5,9 @@ import pytest
from freezegun import freeze_time
from app import statsd_client
# from app.celery.process_sms_client_response_tasks import (
# process_sms_client_response,
# )
from app.celery.process_sms_client_response_tasks import (
process_sms_client_response,
)
from app.clients import ClientException
from app.models import NOTIFICATION_TECHNICAL_FAILURE
tests/app/celery/test_provider_tasks.py
+1
View File
@@ -126,6 +126,7 @@ def test_should_add_to_retry_queue_if_notification_not_found_in_deliver_email_ta
app.delivery.send_to_providers.send_email_to_provider.assert_not_called()
app.celery.provider_tasks.deliver_email.retry.assert_called_with(queue="retry-tasks")
@pytest.mark.skip(reason="Needs updating for TTS: Failing for unknown reason")
@pytest.mark.parametrize(
'exception_class', [
tests/app/celery/test_reporting_tasks.py
+1
View File
@@ -290,6 +290,7 @@ def test_create_nightly_billing_for_day_different_sent_by(
assert record.billable_units == 1
assert record.rate_multiplier == 1.0
@pytest.mark.skip(reason="Needs updating for TTS: Remove mail")
def test_create_nightly_billing_for_day_different_letter_postage(
notify_db_session,
tests/app/clients/test_sms.py
+2
View File
@@ -15,6 +15,7 @@ def fake_client(notify_api):
fake_client.init_app(notify_api, statsd_client)
return fake_client
@pytest.mark.skip(reason="Needs updating for TTS: New SMS client")
def test_send_sms(fake_client, mocker):
mock_send = mocker.patch.object(fake_client, 'try_send_sms')
@@ -31,6 +32,7 @@ def test_send_sms(fake_client, mocker):
'to', 'content', 'reference', False, 'testing'
)
@pytest.mark.skip(reason="Needs updating for TTS: New SMS client")
def test_send_sms_error(fake_client, mocker):
mocker.patch.object(
tests/app/dao/test_provider_details_dao.py
+2
View File
@@ -144,6 +144,7 @@ def test_adjust_provider_priority_sets_priority(
assert mmg_provider.created_by.id == notify_user.id
assert mmg_provider.priority == 50
@pytest.mark.skip(reason="Needs updating for TTS: MMG removal")
@freeze_time('2016-01-01 00:30')
def test_adjust_provider_priority_adds_history(
@@ -172,6 +173,7 @@ def test_adjust_provider_priority_adds_history(
assert updated_provider_history_rows[0].version - old_provider_history_rows[0].version == 1
assert updated_provider_history_rows[0].priority == 50
@pytest.mark.skip(reason="Needs updating for TTS: MMG removal")
@freeze_time('2016-01-01 01:00')
def test_get_sms_providers_for_update_returns_providers(restore_provider_details):
tests/app/delivery/test_send_to_providers.py
+3
View File
@@ -43,6 +43,7 @@ def setup_function(_function):
# state of the cache is not shared between tests.
send_to_providers.provider_cache.clear()
@pytest.mark.skip(reason="Needs updating for TTS: Update with new providers")
def test_provider_to_use_should_return_random_provider(mocker, notify_db_session):
mmg = get_provider_details_by_identifier('mmg')
@@ -72,6 +73,7 @@ def test_provider_to_use_should_cache_repeated_calls(mocker, notify_db_session):
assert all(result == results[0] for result in results)
assert len(mock_choices.call_args_list) == 1
@pytest.mark.skip(reason="Needs updating for TTS: Update with new providers")
@pytest.mark.parametrize('international_provider_priority', (
# Since there’s only one international provider it should always
@@ -592,6 +594,7 @@ def test_should_not_update_notification_if_research_mode_on_exception(
assert persisted_notification.billable_units == 0
assert update_mock.called
@pytest.mark.skip(reason="Needs updating for TTS: Update with new providers")
@pytest.mark.parametrize("starting_status, expected_status", [
("delivered", "delivered"),
tests/app/letters/test_letter_utils.py
+1 -1
View File
@@ -31,8 +31,8 @@ from tests.app.db import create_notification
FROZEN_DATE_TIME = "2018-03-14 17:00:00"
pytest.skip(reason="Skipping letter-related functionality for now", allow_module_level=True)
@pytest.skip(reason="Skipping letter-related functionality for now", allow_module_level=True)
@pytest.fixture(name='sample_precompiled_letter_notification')
def _sample_precompiled_letter_notification(sample_letter_notification):
sample_letter_notification.template.hidden = True
tests/app/notifications/test_notifications_sms_callbacks.py
+8
View File
@@ -17,6 +17,7 @@ def mmg_post(client, data):
data=data,
headers=[('Content-Type', 'application/json')])
@pytest.mark.skip(reason="Needs updating for TTS: Firetext removal")
def test_firetext_callback_should_not_need_auth(client, mocker):
mocker.patch('app.notifications.notifications_sms_callback.process_sms_client_response')
@@ -36,6 +37,7 @@ def test_firetext_callback_should_return_400_if_empty_reference(client, mocker):
assert json_resp['result'] == 'error'
assert json_resp['message'] == ['Firetext callback failed: reference missing']
@pytest.mark.skip(reason="Needs updating for TTS: Firetext removal")
def test_firetext_callback_should_return_400_if_no_reference(client, mocker):
data = 'mobile=441234123123&status=0&time=2016-03-10 14:17:00'
@@ -45,6 +47,7 @@ def test_firetext_callback_should_return_400_if_no_reference(client, mocker):
assert json_resp['result'] == 'error'
assert json_resp['message'] == ['Firetext callback failed: reference missing']
@pytest.mark.skip(reason="Needs updating for TTS: Firetext removal")
def test_firetext_callback_should_return_400_if_no_status(client, mocker):
data = 'mobile=441234123123&time=2016-03-10 14:17:00&reference=notification_id'
@@ -54,6 +57,7 @@ def test_firetext_callback_should_return_400_if_no_status(client, mocker):
assert json_resp['result'] == 'error'
assert json_resp['message'] == ['Firetext callback failed: status missing']
@pytest.mark.skip(reason="Needs updating for TTS: Firetext removal")
def test_firetext_callback_should_return_200_and_call_task_with_valid_data(client, mocker):
mock_celery = mocker.patch(
@@ -70,6 +74,7 @@ def test_firetext_callback_should_return_200_and_call_task_with_valid_data(clien
queue='sms-callbacks',
)
@pytest.mark.skip(reason="Needs updating for TTS: Firetext removal")
def test_firetext_callback_including_a_code_should_return_200_and_call_task_with_valid_data(client, mocker):
mock_celery = mocker.patch(
@@ -86,6 +91,7 @@ def test_firetext_callback_including_a_code_should_return_200_and_call_task_with
queue='sms-callbacks',
)
@pytest.mark.skip(reason="Needs updating for TTS: MMG removal")
def test_mmg_callback_should_not_need_auth(client, mocker, sample_notification):
mocker.patch('app.notifications.notifications_sms_callback.process_sms_client_response')
@@ -98,6 +104,7 @@ def test_mmg_callback_should_not_need_auth(client, mocker, sample_notification):
response = mmg_post(client, data)
assert response.status_code == 200
@pytest.mark.skip(reason="Needs updating for TTS: MMG removal")
def test_process_mmg_response_returns_400_for_malformed_data(client):
data = json.dumps({"reference": "mmg_reference",
@@ -114,6 +121,7 @@ def test_process_mmg_response_returns_400_for_malformed_data(client):
assert "{} callback failed: {} missing".format('MMG', 'status') in json_data['message']
assert "{} callback failed: {} missing".format('MMG', 'CID') in json_data['message']
@pytest.mark.skip(reason="Needs updating for TTS: MMG removal")
def test_mmg_callback_should_return_200_and_call_task_with_valid_data(client, mocker):
mock_celery = mocker.patch(
tests/app/v2/notifications/test_post_notifications.py
+2
View File
@@ -238,6 +238,7 @@ def test_should_cache_template_lookups_in_memory(mocker, client, sample_template
]
assert Notification.query.count() == 5
@pytest.mark.skip(reason="Needs updating for TTS: cloud.gov redis fails, local docker works, mock redis fails")
def test_should_cache_template_and_service_in_redis(mocker, client, sample_template):
@@ -288,6 +289,7 @@ def test_should_cache_template_and_service_in_redis(mocker, client, sample_templ
assert json.loads(templates_call[0][1]) == {'data': template_dict}
assert templates_call[1]['ex'] == 604_800
@pytest.mark.skip(reason="Needs updating for TTS: cloud.gov redis fails, local docker works, mock redis fails")
def test_should_return_template_if_found_in_redis(mocker, client, sample_template):