darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-02-02 09:26:08 -05:00
Code Issues Packages Projects Releases Wiki Activity

make 500s change priorities quicker

Browse Source 
it's not acceptable for a constantly failing provider to take 50 minutes
to drain (5x reducing priority by 10). But similarly, we need _some_
delay, or a handful of concurrent failures will completely turn off a
provider, rendering the whole excercise kinda pointless. Setting the
delay before it tries to reduce priority again to one minute is nice
because it means that if one request times out and returns 502, then any
other requests that are in flight at that time will time out before the
one minute is up and not switch, but any requests made after the switch
that take sixty seconds to time out will affect it.
This commit is contained in:
Leo Hemsted
2019-11-20 17:23:39 +00:00
parent 2d7bf664f5
commit f7fbd6de5b
6 changed files with 28 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
tests/app/dao/test_provider_details_dao.py
View File

@@ -1,6 +1,6 @@
import pytest
from datetime import datetime
from datetime import datetime, timedelta
from freezegun import freeze_time
from sqlalchemy.sql import desc
@@ -153,7 +153,7 @@ def test_reduce_sms_provider_priority_switches_provider(
ProviderDetails.query.filter(ProviderDetails.notification_type == 'sms').update({'updated_at': datetime.min})
# switch away from mmg. currently both 50/50
dao_reduce_sms_provider_priority('mmg')
dao_reduce_sms_provider_priority('mmg', time_threshold=timedelta(minutes=10))
assert firetext.priority == expected_priorities['firetext']
assert mmg.priority == expected_priorities['mmg']
@@ -177,7 +177,7 @@ def test_reduce_sms_provider_priority_adds_rows_to_history_table(
desc(ProviderDetailsHistory.version)
).all()
dao_reduce_sms_provider_priority(mmg.identifier)
dao_reduce_sms_provider_priority(mmg.identifier, time_threshold=timedelta(minutes=10))
updated_provider_history_rows = ProviderDetailsHistory.query.filter(
ProviderDetailsHistory.id == mmg.id
@@ -197,9 +197,11 @@ def test_reduce_sms_provider_priority_does_nothing_if_providers_have_recently_ch
mock_is_slow = mocker.patch('app.celery.scheduled_tasks.is_delivery_slow_for_providers')
mock_reduce = mocker.patch('app.celery.scheduled_tasks.dao_reduce_sms_provider_priority')
ProviderDetails.query.filter(ProviderDetails.identifier == 'firetext').update({'updated_at': datetime.min})
ProviderDetails.query.filter(ProviderDetails.identifier == 'mmg').update({'updated_at': datetime.utcnow()})
ProviderDetails.query.filter(ProviderDetails.identifier == 'mmg').update(
{'updated_at': datetime.utcnow() - timedelta(minutes=4, seconds=59)}
)
dao_reduce_sms_provider_priority('firetext')
dao_reduce_sms_provider_priority('firetext', time_threshold=timedelta(minutes=5))
assert mock_is_slow.called is False
assert mock_reduce.called is False