darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2025-12-15 09:42:38 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2592 from alphagov/reporting-worker

Browse Source 
Add reporting worker
This commit is contained in:
Leo Hemsted
2019-08-15 17:22:27 +01:00
committed by GitHub
parent 689d76d820 e5c76ffda7
commit 92d78956be
6 changed files with 25 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
app/celery/reporting_tasks.py
View File

@@ -24,13 +24,13 @@ from app.celery.nightly_tasks import (
@statsd(namespace="tasks")
def create_nightly_billing(day_start=None):
# day_start is a datetime.date() object. e.g.
# up to 10 days of data counting back from day_start is consolidated
# up to 4 days of data counting back from day_start is consolidated
if day_start is None:
day_start = convert_utc_to_bst(datetime.utcnow()).date() - timedelta(days=1)
else:
# When calling the task its a string in the format of "YYYY-MM-DD"
day_start = datetime.strptime(day_start, "%Y-%m-%d").date()
for i in range(0, 10):
for i in range(0, 4):
process_day = day_start - timedelta(days=i)
transit_data = fetch_billing_data_for_day(process_day=process_day)
@@ -53,7 +53,7 @@ def create_nightly_notification_status(day_start=None):
else:
# When calling the task its a string in the format of "YYYY-MM-DD"
day_start = datetime.strptime(day_start, "%Y-%m-%d").date()
for i in range(0, 10):
for i in range(0, 4):
process_day = day_start - timedelta(days=i)
transit_data = fetch_notification_status_for_day(process_day=process_day)

8
app/config.py
View File

@@ -20,7 +20,7 @@ class QueueNames(object):
SEND_SMS = 'send-sms-tasks'
SEND_EMAIL = 'send-email-tasks'
RESEARCH_MODE = 'research-mode-tasks'
STATISTICS = 'statistics-tasks'
REPORTING = 'reporting-tasks'
JOBS = 'job-tasks'
RETRY = 'retry-tasks'
NOTIFY = 'notify-internal-tasks'
@@ -39,7 +39,7 @@ class QueueNames(object):
QueueNames.SEND_SMS,
QueueNames.SEND_EMAIL,
QueueNames.RESEARCH_MODE,
QueueNames.STATISTICS,
QueueNames.REPORTING,
QueueNames.JOBS,
QueueNames.RETRY,
QueueNames.NOTIFY,
@@ -215,12 +215,12 @@ class Config(object):
'create-nightly-billing': {
'task': 'create-nightly-billing',
'schedule': crontab(hour=0, minute=15),
'options': {'queue': QueueNames.PERIODIC}
'options': {'queue': QueueNames.REPORTING}
},
'create-nightly-notification-status': {
'task': 'create-nightly-notification-status',
'schedule': crontab(hour=0, minute=30), # after 'timeout-sending-notifications'
'options': {'queue': QueueNames.PERIODIC}
'options': {'queue': QueueNames.REPORTING}
},
'delete-inbound-sms': {
'task': 'delete-inbound-sms',

1
manifest.yml.j2
View File

@@ -13,6 +13,7 @@
'notify-delivery-worker-research': {},
'notify-delivery-worker-sender': {'disk_quota': '2G', 'memory': '3G'},
'notify-delivery-worker-periodic': {},
'notify-delivery-worker-reporting': {},
'notify-delivery-worker-priority': {},
'notify-delivery-worker-letters': {},
'notify-delivery-worker-retry-tasks': {},

6
scripts/paas_app_wrapper.sh
View File

@@ -26,7 +26,11 @@ case $NOTIFY_APP_NAME in
;;
delivery-worker-periodic)
scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=2 \
-Q periodic-tasks,statistics-tasks 2> /dev/null
-Q periodic-tasks 2> /dev/null
;;
delivery-worker-reporting)
scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 \
-Q reporting-tasks 2> /dev/null
;;
delivery-worker-priority)
scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=5 \

24
tests/app/celery/test_reporting_tasks.py
View File

@@ -450,14 +450,10 @@ def test_create_nightly_notification_status(notify_db_session, mocker):
create_notification(template=first_template, status='delivered', created_at=datetime.utcnow() - timedelta(days=10))
create_notification(template=second_template, status='temporary-failure')
create_notification(template=second_template, status='temporary-failure',
created_at=datetime.utcnow() - timedelta(days=1))
create_notification(template=second_template, status='temporary-failure',
created_at=datetime.utcnow() - timedelta(days=2))
create_notification(template=second_template, status='temporary-failure',
created_at=datetime.utcnow() - timedelta(days=10))
create_notification(template=second_template, status='temporary-failure',
created_at=datetime.utcnow() - timedelta(days=10))
create_notification(template=second_template, status='temporary-failure', created_at=datetime.utcnow() - timedelta(days=1)) # noqa
create_notification(template=second_template, status='temporary-failure', created_at=datetime.utcnow() - timedelta(days=2)) # noqa
create_notification(template=second_template, status='temporary-failure', created_at=datetime.utcnow() - timedelta(days=10)) # noqa
create_notification(template=second_template, status='temporary-failure', created_at=datetime.utcnow() - timedelta(days=10)) # noqa
create_notification(template=third_template, status='created')
create_notification(template=third_template, status='created', created_at=datetime.utcnow() - timedelta(days=1))
@@ -470,12 +466,14 @@ def test_create_nightly_notification_status(notify_db_session, mocker):
create_nightly_notification_status()
new_data = FactNotificationStatus.query.order_by(
FactNotificationStatus.bst_date,
FactNotificationStatus.notification_type
).all()
assert len(new_data) == 9
assert str(new_data[0].bst_date) == datetime.strftime(datetime.utcnow() - timedelta(days=10), "%Y-%m-%d")
assert str(new_data[3].bst_date) == datetime.strftime(datetime.utcnow() - timedelta(days=2), "%Y-%m-%d")
assert str(new_data[6].bst_date) == datetime.strftime(datetime.utcnow() - timedelta(days=1), "%Y-%m-%d")
assert len(new_data) == 6 #
assert str(new_data[0].bst_date) == datetime.strftime(datetime.utcnow() - timedelta(days=2), "%Y-%m-%d")
assert str(new_data[1].bst_date) == datetime.strftime(datetime.utcnow() - timedelta(days=2), "%Y-%m-%d")
assert str(new_data[2].bst_date) == datetime.strftime(datetime.utcnow() - timedelta(days=2), "%Y-%m-%d")
assert str(new_data[3].bst_date) == datetime.strftime(datetime.utcnow() - timedelta(days=1), "%Y-%m-%d")
assert str(new_data[4].bst_date) == datetime.strftime(datetime.utcnow() - timedelta(days=1), "%Y-%m-%d")
assert str(new_data[5].bst_date) == datetime.strftime(datetime.utcnow() - timedelta(days=1), "%Y-%m-%d")
for mock in mocks:
mock.apply_async.assert_called_once_with(queue='periodic-tasks')

2
tests/app/test_config.py
View File

@@ -71,7 +71,7 @@ def test_queue_names_all_queues_correct():
QueueNames.SEND_SMS,
QueueNames.SEND_EMAIL,
QueueNames.RESEARCH_MODE,
QueueNames.STATISTICS,
QueueNames.REPORTING,
QueueNames.JOBS,
QueueNames.RETRY,
QueueNames.NOTIFY,