darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-02-03 18:01:08 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into schedule-api-notification

Browse Source 
Conflicts:
	app/celery/scheduled_tasks.py
	tests/app/celery/test_scheduled_tasks.py
This commit is contained in:
Rebecca Law
2017-05-24 13:21:22 +01:00
parent 53efb4e033 8a2715e3e1
commit 96b0e42959
8 changed files with 194 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
app/celery/scheduled_tasks.py
View File

@@ -12,9 +12,9 @@ from app import performance_platform_client
from app.dao.invited_user_dao import delete_invitations_created_more_than_two_days_ago
from app.dao.jobs_dao import dao_set_scheduled_jobs_to_pending, dao_get_jobs_older_than_limited_by
from app.dao.notifications_dao import (
delete_notifications_created_more_than_a_week_ago,
dao_timeout_notifications,
is_delivery_slow_for_provider,
delete_notifications_created_more_than_a_week_ago_by_type,
dao_get_scheduled_notifications,
set_scheduled_notification_to_processed)
from app.dao.statistics_dao import dao_timeout_job_statistics
@@ -78,42 +78,60 @@ def delete_verify_codes():
raise
@notify_celery.task(name="delete-successful-notifications")
@notify_celery.task(name="delete-sms-notifications")
@statsd(namespace="tasks")
def delete_successful_notifications():
def delete_sms_notifications_older_than_seven_days():
try:
start = datetime.utcnow()
deleted = delete_notifications_created_more_than_a_week_ago('delivered')
deleted = delete_notifications_created_more_than_a_week_ago_by_type('sms')
current_app.logger.info(
"Delete job started {} finished {} deleted {} successful notifications".format(
"Delete {} job started {} finished {} deleted {} sms notifications".format(
'sms',
start,
datetime.utcnow(),
deleted
)
)
except SQLAlchemyError as e:
current_app.logger.exception("Failed to delete successful notifications")
current_app.logger.exception("Failed to delete sms notifications")
raise
@notify_celery.task(name="delete-failed-notifications")
@notify_celery.task(name="delete-email-notifications")
@statsd(namespace="tasks")
def delete_failed_notifications():
def delete_email_notifications_older_than_seven_days():
try:
start = datetime.utcnow()
deleted = delete_notifications_created_more_than_a_week_ago('failed')
deleted += delete_notifications_created_more_than_a_week_ago('technical-failure')
deleted += delete_notifications_created_more_than_a_week_ago('temporary-failure')
deleted += delete_notifications_created_more_than_a_week_ago('permanent-failure')
deleted = delete_notifications_created_more_than_a_week_ago_by_type('email')
current_app.logger.info(
"Delete job started {} finished {} deleted {} failed notifications".format(
"Delete {} job started {} finished {} deleted {} email notifications".format(
'email',
start,
datetime.utcnow(),
deleted
)
)
except SQLAlchemyError as e:
current_app.logger.exception("Failed to delete failed notifications")
current_app.logger.exception("Failed to delete sms notifications")
raise
@notify_celery.task(name="delete-letter-notifications")
@statsd(namespace="tasks")
def delete_letter_notifications_older_than_seven_days():
try:
start = datetime.utcnow()
deleted = delete_notifications_created_more_than_a_week_ago_by_type('letter')
current_app.logger.info(
"Delete {} job started {} finished {} deleted {} letter notifications".format(
'letter',
start,
datetime.utcnow(),
deleted
)
)
except SQLAlchemyError as e:
current_app.logger.exception("Failed to delete sms notifications")
raise

31
app/commands.py
View File

@@ -2,7 +2,10 @@ import uuid
from datetime import datetime
from decimal import Decimal
from flask.ext.script import Command, Manager, Option
from app.models import (PROVIDERS, Service, User)
from app import db
from app.models import (PROVIDERS, Service, User, NotificationHistory)
from app.dao.services_dao import (
delete_service_and_all_associated_db_objects,
dao_fetch_all_services_by_user
@@ -60,3 +63,29 @@ class PurgeFunctionalTestDataCommand(Command):
else:
delete_user_verify_codes(usr)
delete_model_user(usr)
class CustomDbScript(Command):
def run(self):
self.update_notification_international_flag()
def update_notification_international_flag(self):
# 250,000 rows takes 30 seconds to update.
subq = "select id from notifications where international is null limit 250000"
update = "update notifications set international = False where id in ({})".format(subq)
result = db.session.execute(subq).fetchall()
while len(result) > 0:
db.session.execute(update)
print('commit 250000 updates at {}'.format(datetime.utcnow()))
db.session.commit()
result = db.session.execute(subq).fetchall()
# Now update notification_history
subq_history = "select id from notification_history where international is null limit 250000"
update_history = "update notification_history set international = False where id in ({})".format(subq_history)
result_history = db.session.execute(subq_history).fetchall()
while len(result_history) > 0:
db.session.execute(update_history)
print('commit 250000 updates at {}'.format(datetime.utcnow()))
db.session.commit()
result_history = db.session.execute(subq_history).fetchall()

15
app/config.py
View File

@@ -124,14 +124,19 @@ class Config(object):
'schedule': timedelta(minutes=66),
'options': {'queue': 'periodic'}
},
'delete-failed-notifications': {
'task': 'delete-failed-notifications',
'delete-sms-notifications': {
'task': 'delete-sms-notifications',
'schedule': crontab(minute=0, hour=0),
'options': {'queue': 'periodic'}
},
'delete-successful-notifications': {
'task': 'delete-successful-notifications',
'schedule': crontab(minute=0, hour=1),
'delete-email-notifications': {
'task': 'delete-email-notifications',
'schedule': crontab(minute=20, hour=0),
'options': {'queue': 'periodic'}
},
'delete-letter-notifications': {
'task': 'delete-letter-notifications',
'schedule': crontab(minute=40, hour=0),
'options': {'queue': 'periodic'}
},
'send-daily-performance-platform-stats': {

4
app/dao/notifications_dao.py
View File

@@ -351,11 +351,11 @@ def _filter_query(query, filter_dict=None):
@statsd(namespace="dao")
def delete_notifications_created_more_than_a_week_ago(status):
def delete_notifications_created_more_than_a_week_ago_by_type(notification_type):
seven_days_ago = date.today() - timedelta(days=7)
deleted = db.session.query(Notification).filter(
func.date(Notification.created_at) < seven_days_ago,
Notification.status == status,
Notification.notification_type == notification_type,
).delete(synchronize_session='fetch')
db.session.commit()
return deleted

1
application.py
View File

@@ -15,6 +15,7 @@ migrate = Migrate(application, db)
manager.add_command('db', MigrateCommand)
manager.add_command('create_provider_rate', commands.CreateProviderRateCommand)
manager.add_command('purge_functional_test_data', commands.PurgeFunctionalTestDataCommand)
manager.add_command('custom_db_script', commands.CustomDbScript)
@manager.command

1
server_commands.py
View File

@@ -25,6 +25,7 @@ manager = Manager(application)
migrate = Migrate(application, db)
manager.add_command('db', MigrateCommand)
manager.add_command('purge_functional_test_data', commands.PurgeFunctionalTestDataCommand)
manager.add_command('custom_db_script', commands.CustomDbScript)
if __name__ == '__main__':
manager.run()

32
tests/app/celery/test_scheduled_tasks.py
View File

@@ -5,13 +5,14 @@ from functools import partial
from flask import current_app
from freezegun import freeze_time
from app.celery.scheduled_tasks import s3, timeout_job_statistics, send_scheduled_notifications
from app.celery.scheduled_tasks import s3, timeout_job_statistics, delete_sms_notifications_older_than_seven_days, \
delete_letter_notifications_older_than_seven_days, delete_email_notifications_older_than_seven_days, \
send_scheduled_notifications
from app.celery import scheduled_tasks
from app.celery.scheduled_tasks import (
delete_verify_codes,
remove_csv_files,
delete_successful_notifications,
delete_failed_notifications,
delete_notifications_created_more_than_a_week_ago_by_type,
delete_invitations,
timeout_notifications,
run_scheduled_jobs,
@@ -71,8 +72,7 @@ def prepare_current_provider(restore_provider_details):
def test_should_have_decorated_tasks_functions():
assert delete_verify_codes.__wrapped__.__name__ == 'delete_verify_codes'
assert delete_successful_notifications.__wrapped__.__name__ == 'delete_successful_notifications'
assert delete_failed_notifications.__wrapped__.__name__ == 'delete_failed_notifications'
assert delete_notifications_created_more_than_a_week_ago_by_type.__wrapped__.__name__ == 'delete_notifications_created_more_than_a_week_ago_by_type' # noqa
assert timeout_notifications.__wrapped__.__name__ == 'timeout_notifications'
assert delete_invitations.__wrapped__.__name__ == 'delete_invitations'
assert run_scheduled_jobs.__wrapped__.__name__ == 'run_scheduled_jobs'
@@ -82,16 +82,22 @@ def test_should_have_decorated_tasks_functions():
'switch_current_sms_provider_on_slow_delivery'
def test_should_call_delete_successful_notifications_more_than_week_in_task(notify_api, mocker):
mocked = mocker.patch('app.celery.scheduled_tasks.delete_notifications_created_more_than_a_week_ago')
delete_successful_notifications()
mocked.assert_called_once_with('delivered')
def test_should_call_delete_sms_notifications_more_than_week_in_task(notify_api, mocker):
mocked = mocker.patch('app.celery.scheduled_tasks.delete_notifications_created_more_than_a_week_ago_by_type')
delete_sms_notifications_older_than_seven_days()
mocked.assert_called_once_with('sms')
def test_should_call_delete_failed_notifications_more_than_week_in_task(notify_api, mocker):
mocker.patch('app.celery.scheduled_tasks.delete_notifications_created_more_than_a_week_ago')
delete_failed_notifications()
assert scheduled_tasks.delete_notifications_created_more_than_a_week_ago.call_count == 4
def test_should_call_delete_email_notifications_more_than_week_in_task(notify_api, mocker):
mocked = mocker.patch('app.celery.scheduled_tasks.delete_notifications_created_more_than_a_week_ago_by_type')
delete_email_notifications_older_than_seven_days()
mocked.assert_called_once_with('email')
def test_should_call_delete_letter_notifications_more_than_week_in_task(notify_api, mocker):
mocked = mocker.patch('app.celery.scheduled_tasks.delete_notifications_created_more_than_a_week_ago_by_type')
delete_letter_notifications_older_than_seven_days()
mocked.assert_called_once_with('letter')
def test_should_call_delete_codes_on_delete_verify_codes_task(notify_api, mocker):

142
tests/app/dao/test_notification_dao.py
View File

@@ -18,7 +18,8 @@ from app.models import (
NOTIFICATION_SENT,
KEY_TYPE_NORMAL,
KEY_TYPE_TEAM,
KEY_TYPE_TEST)
KEY_TYPE_TEST
)
from app.dao.notifications_dao import (
dao_create_notification,
@@ -27,7 +28,7 @@ from app.dao.notifications_dao import (
dao_get_potential_notification_statistics_for_day,
dao_get_template_usage,
dao_update_notification,
delete_notifications_created_more_than_a_week_ago,
delete_notifications_created_more_than_a_week_ago_by_type,
get_notification_by_id,
get_notification_for_job,
get_notification_billable_unit_count_per_month,
@@ -52,8 +53,8 @@ from tests.app.conftest import (
sample_email_template,
sample_service,
sample_job,
sample_notification_history as create_notification_history
)
sample_notification_history as create_notification_history,
sample_letter_template)
def test_should_have_decorated_notifications_dao_functions():
@@ -69,7 +70,7 @@ def test_should_have_decorated_notifications_dao_functions():
assert get_notification_with_personalisation.__wrapped__.__name__ == 'get_notification_with_personalisation' # noqa
assert get_notifications_for_service.__wrapped__.__name__ == 'get_notifications_for_service' # noqa
assert get_notification_by_id.__wrapped__.__name__ == 'get_notification_by_id' # noqa
assert delete_notifications_created_more_than_a_week_ago.__wrapped__.__name__ == 'delete_notifications_created_more_than_a_week_ago' # noqa
assert delete_notifications_created_more_than_a_week_ago_by_type.__wrapped__.__name__ == 'delete_notifications_created_more_than_a_week_ago_by_type' # noqa
assert dao_delete_notifications_and_history_by_id.__wrapped__.__name__ == 'dao_delete_notifications_and_history_by_id' # noqa
@@ -876,63 +877,118 @@ def test_updating_notification_with_no_notification_status_updates_notification_
assert hist_from_db._status_fkey == 'failed'
@pytest.mark.parametrize('notification_type, expected_sms_count, expected_email_count, expected_letter_count', [
('sms', 8, 10, 10), ('email', 10, 8, 10), ('letter', 10, 10, 8)
])
@freeze_time("2016-01-10 12:00:00.000000")
def test_should_delete_notifications_after_seven_days(notify_db, notify_db_session):
def test_should_delete_notifications_by_type_after_seven_days(
notify_db,
notify_db_session,
sample_service,
notification_type,
expected_sms_count,
expected_email_count,
expected_letter_count
):
assert len(Notification.query.all()) == 0
# create one notification a day between 1st and 10th from 11:00 to 19:00
email_template = sample_email_template(notify_db, notify_db_session, service=sample_service)
sms_template = sample_template(notify_db, notify_db_session, service=sample_service)
letter_template = sample_letter_template(sample_service)
# create one notification a day between 1st and 10th from 11:00 to 19:00 of each type
for i in range(1, 11):
past_date = '2016-01-{0:02d} {0:02d}:00:00.000000'.format(i)
with freeze_time(past_date):
sample_notification(notify_db, notify_db_session, created_at=datetime.utcnow(), status="failed")
sample_notification(
notify_db,
notify_db_session,
created_at=datetime.utcnow(),
status="failed",
service=sample_service,
template=email_template
)
sample_notification(
notify_db,
notify_db_session,
created_at=datetime.utcnow(),
status="failed",
service=sample_service,
template=sms_template
)
sample_notification(
notify_db,
notify_db_session,
created_at=datetime.utcnow(),
status="failed",
service=sample_service,
template=letter_template
)
all_notifications = Notification.query.all()
assert len(all_notifications) == 10
assert len(all_notifications) == 30
# Records from before 3rd should be deleted
delete_notifications_created_more_than_a_week_ago('failed')
remaining_notifications = Notification.query.all()
assert len(remaining_notifications) == 8
for notification in remaining_notifications:
delete_notifications_created_more_than_a_week_ago_by_type(notification_type)
remaining_sms_notifications = Notification.query.filter_by(notification_type='sms').all()
remaining_letter_notifications = Notification.query.filter_by(notification_type='letter').all()
remaining_email_notifications = Notification.query.filter_by(notification_type='email').all()
assert len(remaining_sms_notifications) == expected_sms_count
assert len(remaining_email_notifications) == expected_email_count
assert len(remaining_letter_notifications) == expected_letter_count
if notification_type == 'sms':
notifications_to_check = remaining_sms_notifications
if notification_type == 'email':
notifications_to_check = remaining_email_notifications
if notification_type == 'letter':
notifications_to_check = remaining_letter_notifications
for notification in notifications_to_check:
assert notification.created_at.date() >= date(2016, 1, 3)
@pytest.mark.parametrize('notification_type', ['sms', 'email', 'letter'])
@freeze_time("2016-01-10 12:00:00.000000")
def test_should_not_delete_notification_history(notify_db, notify_db_session):
def test_should_not_delete_notification_history(notify_db, notify_db_session, sample_service, notification_type):
with freeze_time('2016-01-01 12:00'):
notification = sample_notification(notify_db, notify_db_session, created_at=datetime.utcnow(), status="failed")
notification_id = notification.id
email_template = sample_email_template(notify_db, notify_db_session, service=sample_service)
sms_template = sample_template(notify_db, notify_db_session, service=sample_service)
letter_template = sample_letter_template(sample_service)
assert Notification.query.count() == 1
assert NotificationHistory.query.count() == 1
sample_notification(
notify_db,
notify_db_session,
created_at=datetime.utcnow(),
status="failed",
service=sample_service,
template=email_template
)
sample_notification(
notify_db,
notify_db_session,
created_at=datetime.utcnow(),
status="failed",
service=sample_service,
template=sms_template
)
sample_notification(
notify_db,
notify_db_session,
created_at=datetime.utcnow(),
status="failed",
service=sample_service,
template=letter_template
)
delete_notifications_created_more_than_a_week_ago('failed')
assert Notification.query.count() == 3
assert NotificationHistory.query.count() == 3
assert Notification.query.count() == 0
assert NotificationHistory.query.count() == 1
assert NotificationHistory.query.one().id == notification_id
delete_notifications_created_more_than_a_week_ago_by_type(notification_type)
def test_should_not_delete_failed_notifications_before_seven_days(notify_db, notify_db_session):
should_delete = datetime.utcnow() - timedelta(days=8)
do_not_delete = datetime.utcnow() - timedelta(days=7)
sample_notification(notify_db, notify_db_session, created_at=should_delete, status="failed",
to_field="should_delete")
sample_notification(notify_db, notify_db_session, created_at=do_not_delete, status="failed",
to_field="do_not_delete")
assert len(Notification.query.all()) == 2
delete_notifications_created_more_than_a_week_ago('failed')
assert len(Notification.query.all()) == 1
assert Notification.query.first().to == 'do_not_delete'
def test_should_delete_letter_notifications(sample_letter_template):
should_delete = datetime.utcnow() - timedelta(days=8)
create_notification(sample_letter_template, created_at=should_delete)
delete_notifications_created_more_than_a_week_ago('created')
assert len(Notification.query.all()) == 0
assert Notification.query.count() == 2
assert NotificationHistory.query.count() == 3
@freeze_time("2016-01-10")