Merge branch 'master' into raise-400-for-validation-error

app/aws/s3.py

@@ -120,7 +120,7 @@ def get_list_of_files_by_suffix(bucket_name, subfolder='', suffix='', last_modif
 
     for page in page_iterator:
         for obj in page.get('Contents', []):
-            key = obj['Key'].lower()
+            key = obj['Key']
-            if key.endswith(suffix.lower()):
+            if key.lower().endswith(suffix.lower()):
                 if not last_modified or obj['LastModified'] >= last_modified:
                     yield key

app/celery/scheduled_tasks.py

@@ -337,6 +337,7 @@ def delete_dvla_response_files_older_than_seven_days():
 @notify_celery.task(name="raise-alert-if-letter-notifications-still-sending")
 @statsd(namespace="tasks")
 def raise_alert_if_letter_notifications_still_sending():
+
     today = datetime.utcnow().date()
 
     # Do nothing on the weekend
@@ -356,15 +357,21 @@ def raise_alert_if_letter_notifications_still_sending():
     ).count()
 
     if still_sending:
-        deskpro_client.create_ticket(
+        message = "There are {} letters in the 'sending' state from {}".format(
-            subject="Letters still sending",
+            still_sending,
-            message="There are {} letters in the 'sending' state from {}".format(
+            (today - timedelta(days=offset_days)).strftime('%A %d %B')
-                still_sending,
-                (today - timedelta(days=offset_days)).strftime('%A %d %B')
-            ),
-            ticket_type="alert"
         )
 
+        # Only send alerts in production
+        if current_app.config['NOTIFY_ENVIRONMENT'] in ['production', 'test']:
+            deskpro_client.create_ticket(
+                subject="[{}] Letters still sending".format(current_app.config['NOTIFY_ENVIRONMENT']),
+                message=message,
+                ticket_type="alert"
+            )
+        else:
+            current_app.logger.info(message)
+
 
 @notify_celery.task(name="populate_monthly_billing")
 @statsd(namespace="tasks")
@@ -487,40 +494,42 @@ def daily_stats_template_usage_by_month():
 @statsd(namespace="tasks")
 def letter_raise_alert_if_no_ack_file_for_zip():
     # get a list of zip files since yesterday
-    zip_file_list = []
+    zip_file_set = set()
 
     for key in s3.get_list_of_files_by_suffix(bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'],
-                                              subfolder=datetime.utcnow().strftime('%Y-%m-%d'),
+                                              subfolder=datetime.utcnow().strftime('%Y-%m-%d') + '/zips_sent',
-                                              suffix='.zip'):
+                                              suffix='.TXT'):
-        zip_file_list.append(key)
+        zip_file_set.add(key.upper().rstrip('.TXT'))
 
     # get acknowledgement file
-    ack_file_list = []
+    ack_file_set = set()
     # yesterday = datetime.now(tz=pytz.utc) - timedelta(days=1)
     yesterday = datetime.utcnow() - timedelta(days=1)
     for key in s3.get_list_of_files_by_suffix(bucket_name=current_app.config['DVLA_RESPONSE_BUCKET_NAME'],
                                               subfolder='root/dispatch', suffix='.ACK.txt', last_modified=yesterday):
-        ack_file_list.append(key)
+        ack_file_set.add(key)
 
     today_str = datetime.utcnow().strftime('%Y%m%d')
-    zip_not_today = []
 
-    for key in ack_file_list:
+    ack_content_set = set()
+    for key in ack_file_set:
         if today_str in key:
             content = s3.get_s3_file(current_app.config['DVLA_RESPONSE_BUCKET_NAME'], key)
             for zip_file in content.split('\n'):    # each line
                 s = zip_file.split('|')
-                for zf in zip_file_list:
+                ack_content_set.add(s[0].upper())
-                    if s[0].lower() in zf.lower():
-                        zip_file_list.remove(zf)
-                    else:
-                        zip_not_today.append(s[0])
 
-    if zip_file_list:
+    if len(zip_file_set - ack_content_set) > 0:
-
+        deskpro_client.create_ticket(
-        raise NoAckFileReceived(message=zip_file_list)
+            subject="Letter acknowledge error",
-
+            message="Letter acknowledgement file do not contains all zip files sent: {}".format(datetime.utcnow()
-    if zip_not_today:
+                                                                                                .strftime('%Y-%m-%d')),
-        current_app.logger.info(
+            ticket_type='alert'
-            "letter ack contains zip that is not for today {} ".format(zip_not_today)
+        )
+
+        raise NoAckFileReceived(message=str(zip_file_set - ack_content_set))
+
+    if len(ack_content_set - zip_file_set) > 0:
+        current_app.logger.info(
+            "letter ack contains zip that is not for today: {}".format(ack_content_set - zip_file_set)
         )

migrations/versions/0159_add_historical_redact.py

@@ -0,0 +1,42 @@
+"""empty message
+
+Revision ID: 0159_add_historical_redact
+Revises: 0158_remove_rate_limit_default
+Create Date: 2017-01-17 15:00:00.000000
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '0159_add_historical_redact'
+down_revision = '0158_remove_rate_limit_default'
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+from flask import current_app
+
+def upgrade():
+    op.execute(
+        """
+        INSERT INTO template_redacted
+        (
+            template_id,
+            redact_personalisation,
+            updated_at,
+            updated_by_id
+        )
+        SELECT
+            templates.id,
+            false,
+            now(),
+            '{notify_user}'
+        FROM
+            templates
+        LEFT JOIN template_redacted on template_redacted.template_id = templates.id
+        WHERE template_redacted.template_id IS NULL
+        """.format(notify_user=current_app.config['NOTIFY_USER_ID'])
+    )
+
+
+def downgrade():
+    pass

tests/app/celery/test_scheduled_tasks.py

@@ -642,7 +642,7 @@ def test_alert_if_letter_notifications_still_sending(sample_letter_template, moc
     raise_alert_if_letter_notifications_still_sending()
 
     mock_celery.assert_called_once_with(
-        subject="Letters still sending",
+        subject="[test] Letters still sending",
         message="There are 1 letters in the 'sending' state from Tuesday 16 January",
         ticket_type='alert'
     )
@@ -660,7 +660,7 @@ def test_alert_if_letter_notifications_still_sending_only_alerts_sending(sample_
     raise_alert_if_letter_notifications_still_sending()
 
     mock_celery.assert_called_once_with(
-        subject="Letters still sending",
+        subject="[test] Letters still sending",
         message="There are 1 letters in the 'sending' state from Tuesday 16 January",
         ticket_type='alert'
     )
@@ -703,7 +703,7 @@ def test_monday_alert_if_letter_notifications_still_sending_reports_friday_lette
     raise_alert_if_letter_notifications_still_sending()
 
     mock_celery.assert_called_once_with(
-        subject="Letters still sending",
+        subject="[test] Letters still sending",
         message="There are 2 letters in the 'sending' state from Friday 12 January",
         ticket_type='alert'
     )
@@ -1109,16 +1109,16 @@ def test_dao_fetch_monthly_historical_stats_by_template_null_template_id_not_cou
 
 def mock_s3_get_list_match(bucket_name, subfolder='', suffix='', last_modified=None):
 
-    if subfolder == '2018-01-11':
+    if subfolder == '2018-01-11/zips_sent':
-        return ['NOTIFY.20180111175007.ZIP', 'NOTIFY.20180111175008.ZIP']
+        return ['NOTIFY.20180111175007.ZIP.TXT', 'NOTIFY.20180111175008.ZIP.TXT']
     if subfolder == 'root/dispatch':
         return ['root/dispatch/NOTIFY.20180111175733.ACK.txt']
 
 
 def mock_s3_get_list_diff(bucket_name, subfolder='', suffix='', last_modified=None):
-    if subfolder == '2018-01-11':
+    if subfolder == '2018-01-11/zips_sent':
-        return ['NOTIFY.20180111175007.ZIP', 'NOTIFY.20180111175008.ZIP', 'NOTIFY.20180111175009.ZIP',
+        return ['NOTIFY.20180111175007.ZIP.TXT', 'NOTIFY.20180111175008.ZIP.TXT', 'NOTIFY.20180111175009.ZIP.TXT',
-                'NOTIFY.20180111175010.ZIP']
+                'NOTIFY.20180111175010.ZIP.TXT']
     if subfolder == 'root/dispatch':
         return ['root/dispatch/NOTIFY.20180111175733.ACK.txt']
 
@@ -1133,10 +1133,10 @@ def test_letter_not_raise_alert_if_ack_files_match_zip_list(mocker, notify_db):
     letter_raise_alert_if_no_ack_file_for_zip()
 
     yesterday = datetime.utcnow() - timedelta(days=1)
-    subfoldername = datetime.utcnow().strftime('%Y-%m-%d')
+    subfoldername = datetime.utcnow().strftime('%Y-%m-%d') + '/zips_sent'
     assert mock_file_list.call_count == 2
     assert mock_file_list.call_args_list == [
-        call(bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'], subfolder=subfoldername, suffix='.zip'),
+        call(bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'], subfolder=subfoldername, suffix='.TXT'),
         call(bucket_name=current_app.config['DVLA_RESPONSE_BUCKET_NAME'], subfolder='root/dispatch',
              suffix='.ACK.txt', last_modified=yesterday),
     ]
@@ -1149,9 +1149,29 @@ def test_letter_not_raise_alert_if_ack_files_not_match_zip_list(mocker, notify_d
     mock_get_file = mocker.patch("app.aws.s3.get_s3_file",
                                  return_value='NOTIFY.20180111175007.ZIP|20180111175733\n'
                                               'NOTIFY.20180111175008.ZIP|20180111175734')
+    mock_deskpro = mocker.patch("app.celery.scheduled_tasks.deskpro_client.create_ticket")
+
     with pytest.raises(expected_exception=NoAckFileReceived) as e:
         letter_raise_alert_if_no_ack_file_for_zip()
 
-    assert e.value.message == ['NOTIFY.20180111175009.ZIP', 'NOTIFY.20180111175010.ZIP']
+    assert e.value.message == str(set(['NOTIFY.20180111175009.ZIP', 'NOTIFY.20180111175010.ZIP']))
     assert mock_file_list.call_count == 2
     assert mock_get_file.call_count == 1
+    mock_deskpro.assert_called_once_with(
+        subject="Letter acknowledge error",
+        message="Letter acknowledgement file do not contains all zip files sent: 2018-01-11",
+        ticket_type='alert'
+    )
+
+
+@freeze_time('2018-01-11T23:00:00')
+def test_letter_not_raise_alert_if_no_files_do_not_cause_error(mocker, notify_db):
+    mock_file_list = mocker.patch("app.aws.s3.get_list_of_files_by_suffix", side_effect=None)
+    mock_get_file = mocker.patch("app.aws.s3.get_s3_file",
+                                 return_value='NOTIFY.20180111175007.ZIP|20180111175733\n'
+                                              'NOTIFY.20180111175008.ZIP|20180111175734')
+
+    letter_raise_alert_if_no_ack_file_for_zip()
+
+    assert mock_file_list.call_count == 2
+    assert mock_get_file.call_count == 0