From 07aa44ea82ece6935e95eaffe6059d7822b0d237 Mon Sep 17 00:00:00 2001 From: venusbb Date: Tue, 20 Mar 2018 13:53:31 +0000 Subject: [PATCH 1/3] Add import report_tasks to schedule_tasks. --- app/celery/reporting_tasks.py | 8 ++++---- app/celery/scheduled_tasks.py | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/app/celery/reporting_tasks.py b/app/celery/reporting_tasks.py index 37746742b..20aa6c66f 100644 --- a/app/celery/reporting_tasks.py +++ b/app/celery/reporting_tasks.py @@ -1,5 +1,3 @@ -from app import notify_celery -from notifications_utils.statsd_decorators import statsd import random from datetime import datetime, timedelta from app.models import (Notification, @@ -14,6 +12,8 @@ from app.models import (Notification, from app import db from sqlalchemy import func, desc, case from app.dao.dao_utils import transactional +from notifications_utils.statsd_decorators import statsd +from app import notify_celery def get_rate(non_letter_rates, letter_rates, notification_type, date, crown=None, rate_multiplier=None): @@ -26,10 +26,10 @@ def get_rate(non_letter_rates, letter_rates, notification_type, date, crown=None return 0 -@notify_celery.task(bind=True, name="create-nightly-billing", max_retries=15, default_retry_delay=300) +@notify_celery.task(name="create-nightly-billing") @statsd(namespace="tasks") @transactional -def create_nightly_billing(self, day_start=None): +def create_nightly_billing(day_start=None): if day_start is None: day_start = datetime.date(datetime.utcnow()) - timedelta(days=3) # Nightly jobs consolidating last 3 days # Task to be run after mid-night diff --git a/app/celery/scheduled_tasks.py b/app/celery/scheduled_tasks.py index d863dda19..a3494629c 100644 --- a/app/celery/scheduled_tasks.py +++ b/app/celery/scheduled_tasks.py @@ -70,6 +70,7 @@ from app.celery.service_callback_tasks import ( create_encrypted_callback_data, ) import pytz +from app.celery.reporting_tasks import create_nightly_billing # noqa - otherwise task won't start @worker_process_shutdown.connect From cf019864e2a9f8d4b0b41e7f3bf04813c10d3594 Mon Sep 17 00:00:00 2001 From: venusbb Date: Wed, 21 Mar 2018 09:50:34 +0000 Subject: [PATCH 2/3] Preserve 'unknown' in ft_billing for sms if the provider is not known. --- app/celery/reporting_tasks.py | 17 ++++++++--------- tests/app/celery/test_reporting_tasks.py | 2 +- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/app/celery/reporting_tasks.py b/app/celery/reporting_tasks.py index 20aa6c66f..c60afa373 100644 --- a/app/celery/reporting_tasks.py +++ b/app/celery/reporting_tasks.py @@ -1,4 +1,3 @@ -import random from datetime import datetime, timedelta from app.models import (Notification, Rate, @@ -44,14 +43,14 @@ def create_nightly_billing(day_start=None): Notification.template_id, Notification.service_id, Notification.notification_type, - case( - [ - (Notification.notification_type == 'letter', func.coalesce(Notification.sent_by, 'dvla')), - (Notification.notification_type == 'sms', - func.coalesce(Notification.sent_by, random.choice(['mmg', 'firetext']))) - ], - else_='ses' - ).label('sent_by'), # This could be null - this is a bug to be fixed. + func.coalesce(Notification.sent_by, + case( + [ + (Notification.notification_type == 'letter', 'dvla'), + (Notification.notification_type == 'sms', 'unknown'), + (Notification.notification_type == 'email', 'ses') + ]), + ).label('sent_by'), func.coalesce(Notification.rate_multiplier, 1).label('rate_multiplier'), func.coalesce(Notification.international, False).label('international'), func.sum(Notification.billable_units).label('billable_units'), diff --git a/tests/app/celery/test_reporting_tasks.py b/tests/app/celery/test_reporting_tasks.py index 3b3fcdacb..eb4c8453c 100644 --- a/tests/app/celery/test_reporting_tasks.py +++ b/tests/app/celery/test_reporting_tasks.py @@ -261,7 +261,7 @@ def test_create_nightly_billing_null_sent_by_sms( assert record.rate == Decimal(1.33) assert record.billable_units == 1 assert record.rate_multiplier == 1 - assert record.provider in ['mmg', 'firetext'] + assert record.provider == 'unknown' @freeze_time('2018-01-15T03:30:00') From 378feda60323506c4f8196e3246fdd82e4b2bb05 Mon Sep 17 00:00:00 2001 From: venusbb Date: Wed, 21 Mar 2018 10:39:00 +0000 Subject: [PATCH 3/3] put import reporting_tasks in config --- app/celery/scheduled_tasks.py | 1 - app/config.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/app/celery/scheduled_tasks.py b/app/celery/scheduled_tasks.py index a3494629c..d863dda19 100644 --- a/app/celery/scheduled_tasks.py +++ b/app/celery/scheduled_tasks.py @@ -70,7 +70,6 @@ from app.celery.service_callback_tasks import ( create_encrypted_callback_data, ) import pytz -from app.celery.reporting_tasks import create_nightly_billing # noqa - otherwise task won't start @worker_process_shutdown.connect diff --git a/app/config.py b/app/config.py index a69f61c68..febce73c4 100644 --- a/app/config.py +++ b/app/config.py @@ -161,7 +161,7 @@ class Config(object): CELERY_TIMEZONE = 'Europe/London' CELERY_ACCEPT_CONTENT = ['json'] CELERY_TASK_SERIALIZER = 'json' - CELERY_IMPORTS = ('app.celery.tasks', 'app.celery.scheduled_tasks') + CELERY_IMPORTS = ('app.celery.tasks', 'app.celery.scheduled_tasks', 'app.celery.reporting_tasks') CELERYBEAT_SCHEDULE = { 'run-scheduled-jobs': { 'task': 'run-scheduled-jobs',