darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2025-12-24 01:11:38 -05:00
Code Issues Packages Projects Releases Wiki Activity

update switch provider scheduled task

Browse Source 
it now looks at both providers and works out whether to deprioritise
one, rather than binary switching from one to the other. If anything
has altered the priorities in the last ten minutes it won't take any
action. If both providers are slow it also won't take any action.
This commit is contained in:
Leo Hemsted
2019-11-13 17:01:03 +00:00
parent 3d87096353
commit 2a392e7137
2 changed files with 76 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
app/celery/scheduled_tasks.py
View File

@@ -25,19 +25,17 @@ from app.dao.jobs_dao import (
)
from app.dao.jobs_dao import dao_update_job
from app.dao.notifications_dao import (
is_delivery_slow_for_provider,
dao_get_scheduled_notifications,
set_scheduled_notification_to_processed,
notifications_not_yet_sent,
dao_precompiled_letters_still_pending_virus_check,
dao_old_letters_with_created_status,
letters_missing_from_sending_bucket
)
from app.dao.provider_details_dao import (
get_current_provider,
dao_reduce_sms_provider_priority
letters_missing_from_sending_bucket,
is_delivery_slow_for_providers,
)
from app.dao.provider_details_dao import dao_reduce_sms_provider_priority
from app.dao.users_dao import delete_codes_older_created_more_than_a_day_ago
from app.dao.provider_details_dao import get_provider_details_by_notification_type
from app.models import (
Job,
JOB_STATUS_IN_PROGRESS,
@@ -109,27 +107,30 @@ def delete_invitations():
@statsd(namespace="tasks")
def switch_current_sms_provider_on_slow_delivery():
"""
Switch providers if at least 30% of notifications took more than four minutes to be delivered
in the last ten minutes. Search from the time we last switched to the current provider.
Reduce provider's priority if at least 30% of notifications took more than four minutes to be delivered
in the last ten minutes. If both providers are slow, don't do anything. If we changed the providers in the
last ten minutes, then don't update them again either.
"""
current_provider = get_current_provider('sms')
if current_provider.updated_at > datetime.utcnow() - timedelta(minutes=10):
providers = get_provider_details_by_notification_type('sms')
# if something updated recently, don't update again. If the updated_at is null, set it to min time
# just to prevent errors
if any((provider.updated_at or datetime.min) > datetime.utcnow() - timedelta(minutes=10) for provider in providers):
current_app.logger.info("Slow delivery notifications provider switched less than 10 minutes ago.")
return
slow_delivery_notifications = is_delivery_slow_for_providers(
threshold=0.3,
created_at=datetime.utcnow() - timedelta(minutes=10),
delivery_time=timedelta(minutes=4),
)
if slow_delivery_notifications[current_provider]:
current_app.logger.warning(
'Slow delivery notifications detected for provider {}'.format(
current_provider.identifier
)
)
dao_reduce_sms_provider_priority(current_provider.identifier)
# only adjust if some values are true and some are false - ie, don't adjust if all providers are fast or
# all providers are slow
if len(set(slow_delivery_notifications.values())) != 1:
for provider_name, is_slow in slow_delivery_notifications.items():
if is_slow:
current_app.logger.warning('Slow delivery notifications detected for provider {}'.format(provider_name))
dao_reduce_sms_provider_priority(provider_name)
@notify_celery.task(name='check-job-status')