Merge pull request #1894 from alphagov/handle-ses-complaints

Handle ses complaints

app/models.py

@@ -1814,3 +1814,16 @@ class FactNotificationStatus(db.Model):
     notification_count = db.Column(db.Integer(), nullable=False)
     created_at = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow)
     updated_at = db.Column(db.DateTime, nullable=True, onupdate=datetime.datetime.utcnow)
+
+
+class Complaint(db.Model):
+    __tablename__ = 'complaints'
+    id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+    notification_id = db.Column(UUID(as_uuid=True), db.ForeignKey('notification_history.id'),
+                                index=True, nullable=False)
+    service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'), unique=False, index=True, nullable=False)
+    service = db.relationship(Service, backref=db.backref('complaints'))
+    ses_feedback_id = db.Column(db.Text, nullable=True)
+    complaint_type = db.Column(db.Text, nullable=True)
+    complaint_date = db.Column(db.DateTime, nullable=True)
+    created_at = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow)

app/notifications/notifications_ses_callback.py

@@ -33,11 +33,14 @@ def process_ses_response(ses_request):
 
         notification_type = ses_message['notificationType']
         if notification_type == 'Bounce':
-            current_app.logger.info('SES bounce dict: {}'.format(remove_emails_from_bounce(ses_message['bounce'])))
-            if ses_message['bounce']['bounceType'] == 'Permanent':
-                notification_type = ses_message['bounce']['bounceType']  # permanent or not
-            else:
-                notification_type = 'Temporary'
+            notification_type = determine_notification_bounce_type(notification_type, ses_message)
+        elif notification_type == 'Complaint':
+            # Complaints are going to be stored in a table of it's own,
+            # this will no longer update the status of a notification as it does now.
+            remove_emails_from_complaint(ses_message)
+            current_app.logger.info("Complaint from SES: \n{}".format(ses_message))
+            return
+
         try:
             aws_response_dict = get_aws_responses(notification_type)
         except KeyError:
@@ -91,11 +94,24 @@ def process_ses_response(ses_request):
         return error
 
 
+def determine_notification_bounce_type(notification_type, ses_message):
+    current_app.logger.info('SES bounce dict: {}'.format(remove_emails_from_bounce(ses_message['bounce'])))
+    if ses_message['bounce']['bounceType'] == 'Permanent':
+        notification_type = ses_message['bounce']['bounceType']  # permanent or not
+    else:
+        notification_type = 'Temporary'
+    return notification_type
+
+
 def remove_emails_from_bounce(bounce_dict):
     for recip in bounce_dict['bouncedRecipients']:
         recip.pop('emailAddress')
 
 
+def remove_emails_from_complaint(complaint_dict):
+    complaint_dict['complaint'].pop('complainedRecipients')
+
+
 def _check_and_queue_callback_task(notification):
     # queue callback task only if the service_callback_api exists
     service_callback_api = get_service_callback_api_for_service(service_id=notification.service_id)

app/schemas.py

@@ -235,6 +235,7 @@ class ServiceSchema(BaseSchema):
             'monthly_billing',
             'reply_to_email_addresses',
             'letter_contacts',
+            'complaints',
         )
         strict = True
 

migrations/versions/0196_complaints_table_.py

@@ -0,0 +1,36 @@
+"""
+
+Revision ID: 0196_complaints_table
+Revises: 0195_ft_notification_timestamps
+Create Date: 2018-05-31 14:31:36.649544
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+revision = '0196_complaints_table'
+down_revision = '0195_ft_notification_timestamps'
+
+
+def upgrade():
+    op.create_table('complaints',
+    sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False),
+    sa.Column('notification_id', postgresql.UUID(as_uuid=True), nullable=False),
+    sa.Column('service_id', postgresql.UUID(as_uuid=True), nullable=False),
+    sa.Column('ses_feedback_id', sa.Text(), nullable=True),
+    sa.Column('complaint_type', sa.Text(), nullable=True),
+    sa.Column('complaint_date', sa.DateTime(), nullable=True),
+    sa.Column('created_at', sa.DateTime(), nullable=False),
+    sa.ForeignKeyConstraint(['notification_id'], ['notification_history.id'], ),
+    sa.ForeignKeyConstraint(['service_id'], ['services.id'], ),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_complaints_notification_id'), 'complaints', ['notification_id'], unique=False)
+    op.create_index(op.f('ix_complaints_service_id'), 'complaints', ['service_id'], unique=False)
+
+
+def downgrade():
+    op.drop_index(op.f('ix_complaints_service_id'), table_name='complaints')
+    op.drop_index(op.f('ix_complaints_notification_id'), table_name='complaints')
+    op.drop_table('complaints')

tests/app/celery/test_process_ses_receipts_tasks.py

@@ -2,6 +2,7 @@ import json
 from datetime import datetime
 
 from app.celery.process_ses_receipts_tasks import process_ses_results
+from app.notifications.notifications_ses_callback import remove_emails_from_complaint
 
 from tests.app.db import create_notification
 
@@ -32,6 +33,20 @@ def test_process_ses_results_retry_called(notify_db, mocker):
     assert mocked.call_count != 0
 
 
+def test_process_ses_results_in_complaint(notify_db, mocker):
+    mocked = mocker.patch("app.dao.notifications_dao.update_notification_status_by_reference")
+    response = json.loads(ses_complaint_callback())
+    process_ses_results(response=response)
+    assert mocked.call_count == 0
+
+
+def test_remove_emails_from_complaint():
+    test_message = ses_complaint_callback()
+    test_json = json.loads(json.loads(test_message)['Message'])
+    remove_emails_from_complaint(test_json)
+    assert "recipient1@example.com" not in test_json
+
+
 def ses_notification_callback():
     return '{\n  "Type" : "Notification",\n  "MessageId" : "ref1",' \
            '\n  "TopicArn" : "arn:aws:sns:eu-west-1:123456789012:testing",' \
@@ -56,3 +71,18 @@ def ses_notification_callback():
            'dd426d95ee9390147a5624348ee.pem",' \
            '\n  "UnsubscribeURL" : "https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&S' \
            'ubscriptionArn=arn:aws:sns:eu-west-1:302763885840:preview-emails:d6aad3ef-83d6-4cf3-a470-54e2e75916da"\n}'
+
+
+def ses_complaint_callback():
+    """
+    https://docs.aws.amazon.com/ses/latest/DeveloperGuide/notification-contents.html#complaint-object
+    """
+    return '{\n  "Type" : "Notification",\n  "MessageId" : "ref1",' \
+           '\n  "TopicArn" : "arn:aws:sns:eu-west-1:123456789012:testing",' \
+           '\n  "Message" : "{\\"notificationType\\":\\"Complaint\\",' \
+           '\\"complaint\\": {\\"userAgent\\":\\"AnyCompany Feedback Loop (V0.01)\\",' \
+           '\\"complainedRecipients\\":[{\\"emailAddress\\":\\"recipient1@example.com\\"}],' \
+           '\\"complaintFeedbackType\\":\\"abuse\\", ' \
+           '\\"arrivalDate\\":\\"2009-12-03T04:24:21.000-05:00\\", ' \
+           '\\"timestamp\\":\\"2012-05-25T14:59:38.623Z\\", ' \
+           '\\"feedbackId\\":\\"someSESID\\"}}"\n}'