darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2025-12-24 01:11:38 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into raise-400-for-validation-error

Browse Source
This commit is contained in:
Rebecca Law
2018-01-19 13:41:24 +00:00
committed by GitHub
parent 93ce1d2503 db29874948
commit 8a126bbfbb
4 changed files with 111 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
app/aws/s3.py
View File

@@ -120,7 +120,7 @@ def get_list_of_files_by_suffix(bucket_name, subfolder='', suffix='', last_modif
for page in page_iterator:
for obj in page.get('Contents', []):
key = obj['Key'].lower()
if key.endswith(suffix.lower()):
key = obj['Key']
if key.lower().endswith(suffix.lower()):
if not last_modified or obj['LastModified'] >= last_modified:
yield key

63
app/celery/scheduled_tasks.py
View File

@@ -337,6 +337,7 @@ def delete_dvla_response_files_older_than_seven_days():
@notify_celery.task(name="raise-alert-if-letter-notifications-still-sending")
@statsd(namespace="tasks")
def raise_alert_if_letter_notifications_still_sending():
today = datetime.utcnow().date()
# Do nothing on the weekend
@@ -356,15 +357,21 @@ def raise_alert_if_letter_notifications_still_sending():
).count()
if still_sending:
deskpro_client.create_ticket(
subject="Letters still sending",
message="There are {} letters in the 'sending' state from {}".format(
still_sending,
(today - timedelta(days=offset_days)).strftime('%A %d %B')
),
ticket_type="alert"
message = "There are {} letters in the 'sending' state from {}".format(
still_sending,
(today - timedelta(days=offset_days)).strftime('%A %d %B')
)
# Only send alerts in production
if current_app.config['NOTIFY_ENVIRONMENT'] in ['production', 'test']:
deskpro_client.create_ticket(
subject="[{}] Letters still sending".format(current_app.config['NOTIFY_ENVIRONMENT']),
message=message,
ticket_type="alert"
)
else:
current_app.logger.info(message)
@notify_celery.task(name="populate_monthly_billing")
@statsd(namespace="tasks")
@@ -487,40 +494,42 @@ def daily_stats_template_usage_by_month():
@statsd(namespace="tasks")
def letter_raise_alert_if_no_ack_file_for_zip():
# get a list of zip files since yesterday
zip_file_list = []
zip_file_set = set()
for key in s3.get_list_of_files_by_suffix(bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'],
subfolder=datetime.utcnow().strftime('%Y-%m-%d'),
suffix='.zip'):
zip_file_list.append(key)
subfolder=datetime.utcnow().strftime('%Y-%m-%d') + '/zips_sent',
suffix='.TXT'):
zip_file_set.add(key.upper().rstrip('.TXT'))
# get acknowledgement file
ack_file_list = []
ack_file_set = set()
# yesterday = datetime.now(tz=pytz.utc) - timedelta(days=1)
yesterday = datetime.utcnow() - timedelta(days=1)
for key in s3.get_list_of_files_by_suffix(bucket_name=current_app.config['DVLA_RESPONSE_BUCKET_NAME'],
subfolder='root/dispatch', suffix='.ACK.txt', last_modified=yesterday):
ack_file_list.append(key)
ack_file_set.add(key)
today_str = datetime.utcnow().strftime('%Y%m%d')
zip_not_today = []
for key in ack_file_list:
ack_content_set = set()
for key in ack_file_set:
if today_str in key:
content = s3.get_s3_file(current_app.config['DVLA_RESPONSE_BUCKET_NAME'], key)
for zip_file in content.split('\n'): # each line
s = zip_file.split('|')
for zf in zip_file_list:
if s[0].lower() in zf.lower():
zip_file_list.remove(zf)
else:
zip_not_today.append(s[0])
ack_content_set.add(s[0].upper())
if zip_file_list:
raise NoAckFileReceived(message=zip_file_list)
if zip_not_today:
current_app.logger.info(
"letter ack contains zip that is not for today {} ".format(zip_not_today)
if len(zip_file_set - ack_content_set) > 0:
deskpro_client.create_ticket(
subject="Letter acknowledge error",
message="Letter acknowledgement file do not contains all zip files sent: {}".format(datetime.utcnow()
.strftime('%Y-%m-%d')),
ticket_type='alert'
)
raise NoAckFileReceived(message=str(zip_file_set - ack_content_set))
if len(ack_content_set - zip_file_set) > 0:
current_app.logger.info(
"letter ack contains zip that is not for today: {}".format(ack_content_set - zip_file_set)
)

42
migrations/versions/0159_add_historical_redact.py Normal file
View File

@@ -0,0 +1,42 @@
"""empty message
Revision ID: 0159_add_historical_redact
Revises: 0158_remove_rate_limit_default
Create Date: 2017-01-17 15:00:00.000000
"""
# revision identifiers, used by Alembic.
revision = '0159_add_historical_redact'
down_revision = '0158_remove_rate_limit_default'
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
from flask import current_app
def upgrade():
op.execute(
"""
INSERT INTO template_redacted
(
template_id,
redact_personalisation,
updated_at,
updated_by_id
)
SELECT
templates.id,
false,
now(),
'{notify_user}'
FROM
templates
LEFT JOIN template_redacted on template_redacted.template_id = templates.id
WHERE template_redacted.template_id IS NULL
""".format(notify_user=current_app.config['NOTIFY_USER_ID'])
)
def downgrade():
pass

42
tests/app/celery/test_scheduled_tasks.py
View File

@@ -642,7 +642,7 @@ def test_alert_if_letter_notifications_still_sending(sample_letter_template, moc
raise_alert_if_letter_notifications_still_sending()
mock_celery.assert_called_once_with(
subject="Letters still sending",
subject="[test] Letters still sending",
message="There are 1 letters in the 'sending' state from Tuesday 16 January",
ticket_type='alert'
)
@@ -660,7 +660,7 @@ def test_alert_if_letter_notifications_still_sending_only_alerts_sending(sample_
raise_alert_if_letter_notifications_still_sending()
mock_celery.assert_called_once_with(
subject="Letters still sending",
subject="[test] Letters still sending",
message="There are 1 letters in the 'sending' state from Tuesday 16 January",
ticket_type='alert'
)
@@ -703,7 +703,7 @@ def test_monday_alert_if_letter_notifications_still_sending_reports_friday_lette
raise_alert_if_letter_notifications_still_sending()
mock_celery.assert_called_once_with(
subject="Letters still sending",
subject="[test] Letters still sending",
message="There are 2 letters in the 'sending' state from Friday 12 January",
ticket_type='alert'
)
@@ -1109,16 +1109,16 @@ def test_dao_fetch_monthly_historical_stats_by_template_null_template_id_not_cou
def mock_s3_get_list_match(bucket_name, subfolder='', suffix='', last_modified=None):
if subfolder == '2018-01-11':
return ['NOTIFY.20180111175007.ZIP', 'NOTIFY.20180111175008.ZIP']
if subfolder == '2018-01-11/zips_sent':
return ['NOTIFY.20180111175007.ZIP.TXT', 'NOTIFY.20180111175008.ZIP.TXT']
if subfolder == 'root/dispatch':
return ['root/dispatch/NOTIFY.20180111175733.ACK.txt']
def mock_s3_get_list_diff(bucket_name, subfolder='', suffix='', last_modified=None):
if subfolder == '2018-01-11':
return ['NOTIFY.20180111175007.ZIP', 'NOTIFY.20180111175008.ZIP', 'NOTIFY.20180111175009.ZIP',
'NOTIFY.20180111175010.ZIP']
if subfolder == '2018-01-11/zips_sent':
return ['NOTIFY.20180111175007.ZIP.TXT', 'NOTIFY.20180111175008.ZIP.TXT', 'NOTIFY.20180111175009.ZIP.TXT',
'NOTIFY.20180111175010.ZIP.TXT']
if subfolder == 'root/dispatch':
return ['root/dispatch/NOTIFY.20180111175733.ACK.txt']
@@ -1133,10 +1133,10 @@ def test_letter_not_raise_alert_if_ack_files_match_zip_list(mocker, notify_db):
letter_raise_alert_if_no_ack_file_for_zip()
yesterday = datetime.utcnow() - timedelta(days=1)
subfoldername = datetime.utcnow().strftime('%Y-%m-%d')
subfoldername = datetime.utcnow().strftime('%Y-%m-%d') + '/zips_sent'
assert mock_file_list.call_count == 2
assert mock_file_list.call_args_list == [
call(bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'], subfolder=subfoldername, suffix='.zip'),
call(bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'], subfolder=subfoldername, suffix='.TXT'),
call(bucket_name=current_app.config['DVLA_RESPONSE_BUCKET_NAME'], subfolder='root/dispatch',
suffix='.ACK.txt', last_modified=yesterday),
]
@@ -1149,9 +1149,29 @@ def test_letter_not_raise_alert_if_ack_files_not_match_zip_list(mocker, notify_d
mock_get_file = mocker.patch("app.aws.s3.get_s3_file",
return_value='NOTIFY.20180111175007.ZIP|20180111175733\n'
'NOTIFY.20180111175008.ZIP|20180111175734')
mock_deskpro = mocker.patch("app.celery.scheduled_tasks.deskpro_client.create_ticket")
with pytest.raises(expected_exception=NoAckFileReceived) as e:
letter_raise_alert_if_no_ack_file_for_zip()
assert e.value.message == ['NOTIFY.20180111175009.ZIP', 'NOTIFY.20180111175010.ZIP']
assert e.value.message == str(set(['NOTIFY.20180111175009.ZIP', 'NOTIFY.20180111175010.ZIP']))
assert mock_file_list.call_count == 2
assert mock_get_file.call_count == 1
mock_deskpro.assert_called_once_with(
subject="Letter acknowledge error",
message="Letter acknowledgement file do not contains all zip files sent: 2018-01-11",
ticket_type='alert'
)
@freeze_time('2018-01-11T23:00:00')
def test_letter_not_raise_alert_if_no_files_do_not_cause_error(mocker, notify_db):
mock_file_list = mocker.patch("app.aws.s3.get_list_of_files_by_suffix", side_effect=None)
mock_get_file = mocker.patch("app.aws.s3.get_s3_file",
return_value='NOTIFY.20180111175007.ZIP|20180111175733\n'
'NOTIFY.20180111175008.ZIP|20180111175734')
letter_raise_alert_if_no_ack_file_for_zip()
assert mock_file_list.call_count == 2
assert mock_get_file.call_count == 0