diff --git a/app/aws/s3.py b/app/aws/s3.py index e8ef69edc..8020c18a7 100644 --- a/app/aws/s3.py +++ b/app/aws/s3.py @@ -120,7 +120,7 @@ def get_list_of_files_by_suffix(bucket_name, subfolder='', suffix='', last_modif for page in page_iterator: for obj in page.get('Contents', []): - key = obj['Key'].lower() - if key.endswith(suffix.lower()): + key = obj['Key'] + if key.lower().endswith(suffix.lower()): if not last_modified or obj['LastModified'] >= last_modified: yield key diff --git a/app/celery/scheduled_tasks.py b/app/celery/scheduled_tasks.py index 6ebaec87a..9bc8343b3 100644 --- a/app/celery/scheduled_tasks.py +++ b/app/celery/scheduled_tasks.py @@ -337,6 +337,7 @@ def delete_dvla_response_files_older_than_seven_days(): @notify_celery.task(name="raise-alert-if-letter-notifications-still-sending") @statsd(namespace="tasks") def raise_alert_if_letter_notifications_still_sending(): + today = datetime.utcnow().date() # Do nothing on the weekend @@ -356,15 +357,21 @@ def raise_alert_if_letter_notifications_still_sending(): ).count() if still_sending: - deskpro_client.create_ticket( - subject="Letters still sending", - message="There are {} letters in the 'sending' state from {}".format( - still_sending, - (today - timedelta(days=offset_days)).strftime('%A %d %B') - ), - ticket_type="alert" + message = "There are {} letters in the 'sending' state from {}".format( + still_sending, + (today - timedelta(days=offset_days)).strftime('%A %d %B') ) + # Only send alerts in production + if current_app.config['NOTIFY_ENVIRONMENT'] in ['production', 'test']: + deskpro_client.create_ticket( + subject="[{}] Letters still sending".format(current_app.config['NOTIFY_ENVIRONMENT']), + message=message, + ticket_type="alert" + ) + else: + current_app.logger.info(message) + @notify_celery.task(name="populate_monthly_billing") @statsd(namespace="tasks") @@ -487,40 +494,42 @@ def daily_stats_template_usage_by_month(): @statsd(namespace="tasks") def letter_raise_alert_if_no_ack_file_for_zip(): # get a list of zip files since yesterday - zip_file_list = [] + zip_file_set = set() for key in s3.get_list_of_files_by_suffix(bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'], - subfolder=datetime.utcnow().strftime('%Y-%m-%d'), - suffix='.zip'): - zip_file_list.append(key) + subfolder=datetime.utcnow().strftime('%Y-%m-%d') + '/zips_sent', + suffix='.TXT'): + zip_file_set.add(key.upper().rstrip('.TXT')) # get acknowledgement file - ack_file_list = [] + ack_file_set = set() # yesterday = datetime.now(tz=pytz.utc) - timedelta(days=1) yesterday = datetime.utcnow() - timedelta(days=1) for key in s3.get_list_of_files_by_suffix(bucket_name=current_app.config['DVLA_RESPONSE_BUCKET_NAME'], subfolder='root/dispatch', suffix='.ACK.txt', last_modified=yesterday): - ack_file_list.append(key) + ack_file_set.add(key) today_str = datetime.utcnow().strftime('%Y%m%d') - zip_not_today = [] - for key in ack_file_list: + ack_content_set = set() + for key in ack_file_set: if today_str in key: content = s3.get_s3_file(current_app.config['DVLA_RESPONSE_BUCKET_NAME'], key) for zip_file in content.split('\n'): # each line s = zip_file.split('|') - for zf in zip_file_list: - if s[0].lower() in zf.lower(): - zip_file_list.remove(zf) - else: - zip_not_today.append(s[0]) + ack_content_set.add(s[0].upper()) - if zip_file_list: - - raise NoAckFileReceived(message=zip_file_list) - - if zip_not_today: - current_app.logger.info( - "letter ack contains zip that is not for today {} ".format(zip_not_today) + if len(zip_file_set - ack_content_set) > 0: + deskpro_client.create_ticket( + subject="Letter acknowledge error", + message="Letter acknowledgement file do not contains all zip files sent: {}".format(datetime.utcnow() + .strftime('%Y-%m-%d')), + ticket_type='alert' + ) + + raise NoAckFileReceived(message=str(zip_file_set - ack_content_set)) + + if len(ack_content_set - zip_file_set) > 0: + current_app.logger.info( + "letter ack contains zip that is not for today: {}".format(ack_content_set - zip_file_set) ) diff --git a/migrations/versions/0159_add_historical_redact.py b/migrations/versions/0159_add_historical_redact.py new file mode 100644 index 000000000..16ea747ab --- /dev/null +++ b/migrations/versions/0159_add_historical_redact.py @@ -0,0 +1,42 @@ +"""empty message + +Revision ID: 0159_add_historical_redact +Revises: 0158_remove_rate_limit_default +Create Date: 2017-01-17 15:00:00.000000 + +""" + +# revision identifiers, used by Alembic. +revision = '0159_add_historical_redact' +down_revision = '0158_remove_rate_limit_default' + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql +from flask import current_app + +def upgrade(): + op.execute( + """ + INSERT INTO template_redacted + ( + template_id, + redact_personalisation, + updated_at, + updated_by_id + ) + SELECT + templates.id, + false, + now(), + '{notify_user}' + FROM + templates + LEFT JOIN template_redacted on template_redacted.template_id = templates.id + WHERE template_redacted.template_id IS NULL + """.format(notify_user=current_app.config['NOTIFY_USER_ID']) + ) + + +def downgrade(): + pass diff --git a/tests/app/celery/test_scheduled_tasks.py b/tests/app/celery/test_scheduled_tasks.py index 0eb25f2a9..304fb0259 100644 --- a/tests/app/celery/test_scheduled_tasks.py +++ b/tests/app/celery/test_scheduled_tasks.py @@ -642,7 +642,7 @@ def test_alert_if_letter_notifications_still_sending(sample_letter_template, moc raise_alert_if_letter_notifications_still_sending() mock_celery.assert_called_once_with( - subject="Letters still sending", + subject="[test] Letters still sending", message="There are 1 letters in the 'sending' state from Tuesday 16 January", ticket_type='alert' ) @@ -660,7 +660,7 @@ def test_alert_if_letter_notifications_still_sending_only_alerts_sending(sample_ raise_alert_if_letter_notifications_still_sending() mock_celery.assert_called_once_with( - subject="Letters still sending", + subject="[test] Letters still sending", message="There are 1 letters in the 'sending' state from Tuesday 16 January", ticket_type='alert' ) @@ -703,7 +703,7 @@ def test_monday_alert_if_letter_notifications_still_sending_reports_friday_lette raise_alert_if_letter_notifications_still_sending() mock_celery.assert_called_once_with( - subject="Letters still sending", + subject="[test] Letters still sending", message="There are 2 letters in the 'sending' state from Friday 12 January", ticket_type='alert' ) @@ -1109,16 +1109,16 @@ def test_dao_fetch_monthly_historical_stats_by_template_null_template_id_not_cou def mock_s3_get_list_match(bucket_name, subfolder='', suffix='', last_modified=None): - if subfolder == '2018-01-11': - return ['NOTIFY.20180111175007.ZIP', 'NOTIFY.20180111175008.ZIP'] + if subfolder == '2018-01-11/zips_sent': + return ['NOTIFY.20180111175007.ZIP.TXT', 'NOTIFY.20180111175008.ZIP.TXT'] if subfolder == 'root/dispatch': return ['root/dispatch/NOTIFY.20180111175733.ACK.txt'] def mock_s3_get_list_diff(bucket_name, subfolder='', suffix='', last_modified=None): - if subfolder == '2018-01-11': - return ['NOTIFY.20180111175007.ZIP', 'NOTIFY.20180111175008.ZIP', 'NOTIFY.20180111175009.ZIP', - 'NOTIFY.20180111175010.ZIP'] + if subfolder == '2018-01-11/zips_sent': + return ['NOTIFY.20180111175007.ZIP.TXT', 'NOTIFY.20180111175008.ZIP.TXT', 'NOTIFY.20180111175009.ZIP.TXT', + 'NOTIFY.20180111175010.ZIP.TXT'] if subfolder == 'root/dispatch': return ['root/dispatch/NOTIFY.20180111175733.ACK.txt'] @@ -1133,10 +1133,10 @@ def test_letter_not_raise_alert_if_ack_files_match_zip_list(mocker, notify_db): letter_raise_alert_if_no_ack_file_for_zip() yesterday = datetime.utcnow() - timedelta(days=1) - subfoldername = datetime.utcnow().strftime('%Y-%m-%d') + subfoldername = datetime.utcnow().strftime('%Y-%m-%d') + '/zips_sent' assert mock_file_list.call_count == 2 assert mock_file_list.call_args_list == [ - call(bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'], subfolder=subfoldername, suffix='.zip'), + call(bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'], subfolder=subfoldername, suffix='.TXT'), call(bucket_name=current_app.config['DVLA_RESPONSE_BUCKET_NAME'], subfolder='root/dispatch', suffix='.ACK.txt', last_modified=yesterday), ] @@ -1149,9 +1149,29 @@ def test_letter_not_raise_alert_if_ack_files_not_match_zip_list(mocker, notify_d mock_get_file = mocker.patch("app.aws.s3.get_s3_file", return_value='NOTIFY.20180111175007.ZIP|20180111175733\n' 'NOTIFY.20180111175008.ZIP|20180111175734') + mock_deskpro = mocker.patch("app.celery.scheduled_tasks.deskpro_client.create_ticket") + with pytest.raises(expected_exception=NoAckFileReceived) as e: letter_raise_alert_if_no_ack_file_for_zip() - assert e.value.message == ['NOTIFY.20180111175009.ZIP', 'NOTIFY.20180111175010.ZIP'] + assert e.value.message == str(set(['NOTIFY.20180111175009.ZIP', 'NOTIFY.20180111175010.ZIP'])) assert mock_file_list.call_count == 2 assert mock_get_file.call_count == 1 + mock_deskpro.assert_called_once_with( + subject="Letter acknowledge error", + message="Letter acknowledgement file do not contains all zip files sent: 2018-01-11", + ticket_type='alert' + ) + + +@freeze_time('2018-01-11T23:00:00') +def test_letter_not_raise_alert_if_no_files_do_not_cause_error(mocker, notify_db): + mock_file_list = mocker.patch("app.aws.s3.get_list_of_files_by_suffix", side_effect=None) + mock_get_file = mocker.patch("app.aws.s3.get_s3_file", + return_value='NOTIFY.20180111175007.ZIP|20180111175733\n' + 'NOTIFY.20180111175008.ZIP|20180111175734') + + letter_raise_alert_if_no_ack_file_for_zip() + + assert mock_file_list.call_count == 2 + assert mock_get_file.call_count == 0