diff --git a/app/celery/reporting_tasks.py b/app/celery/reporting_tasks.py index b11f08261..0036a076f 100644 --- a/app/celery/reporting_tasks.py +++ b/app/celery/reporting_tasks.py @@ -91,54 +91,58 @@ def create_nightly_notification_status(): yesterday = convert_utc_to_bst(datetime.utcnow()).date() - timedelta(days=1) - for notification_type in [SMS_TYPE, EMAIL_TYPE, LETTER_TYPE]: - days = 10 if notification_type == LETTER_TYPE else 4 + for (service_id,) in db.session.query(Service.id): + for notification_type in [SMS_TYPE, EMAIL_TYPE, LETTER_TYPE]: + days = 10 if notification_type == LETTER_TYPE else 4 - for i in range(days): - process_day = yesterday - timedelta(days=i) + for i in range(days): + process_day = yesterday - timedelta(days=i) - create_nightly_notification_status_for_day.apply_async( - kwargs={'process_day': process_day.isoformat(), 'notification_type': notification_type}, - queue=QueueNames.REPORTING - ) - current_app.logger.info( - f"create-nightly-notification-status-for-day task created " - f"for {notification_type} and {process_day}" - ) + create_nightly_notification_status_for_service_and_day.apply_async( + kwargs={ + 'process_day': process_day.isoformat(), + 'notification_type': notification_type, + 'service_id': service_id, + }, + queue=QueueNames.REPORTING + ) + current_app.logger.info( + f"create-nightly-notification-status-for-day task created " + f"for {service_id}, {notification_type} and {process_day}" + ) -@notify_celery.task(name="create-nightly-notification-status-for-day") -def create_nightly_notification_status_for_day(process_day, notification_type): +@notify_celery.task(name="create-nightly-notification-status-for-service-and-day") +def create_nightly_notification_status_for_service_and_day(process_day, service_id, notification_type): process_day = datetime.strptime(process_day, "%Y-%m-%d").date() current_app.logger.info( f'create-nightly-notification-status-for-day task started ' - f'for {notification_type} and {process_day}' + f'for {service_id}, {notification_type} and {process_day}' ) - for (service_id,) in db.session.query(Service.id): - start = datetime.utcnow() - transit_data = fetch_status_data_for_service_and_day( - process_day=process_day, - notification_type=notification_type, - service_id=service_id, - ) + start = datetime.utcnow() + transit_data = fetch_status_data_for_service_and_day( + process_day=process_day, + notification_type=notification_type, + service_id=service_id, + ) - end = datetime.utcnow() - current_app.logger.info( - f'create-nightly-notification-status-for-day task fetch ' - f'for {process_day} and {notification_type}: ' - f'data fetched in {(end - start).seconds} seconds' - ) + end = datetime.utcnow() + current_app.logger.info( + f'create-nightly-notification-status-for-day task fetch ' + f'for {service_id}, {process_day} and {notification_type}: ' + f'data fetched in {(end - start).seconds} seconds' + ) - update_fact_notification_status( - transit_data=transit_data, - process_day=process_day, - notification_type=notification_type, - service_id=service_id - ) + update_fact_notification_status( + transit_data=transit_data, + process_day=process_day, + notification_type=notification_type, + service_id=service_id + ) - current_app.logger.info( - f'create-nightly-notification-status-for-day task finished ' - f'for {process_day} and {notification_type}: ' - f'{len(transit_data)} rows updated' - ) + current_app.logger.info( + f'create-nightly-notification-status-for-day task finished ' + f'for {service_id}, {process_day} and {notification_type}: ' + f'{len(transit_data)} rows updated' + ) diff --git a/tests/app/celery/test_reporting_tasks.py b/tests/app/celery/test_reporting_tasks.py index dacab4db0..9e0345200 100644 --- a/tests/app/celery/test_reporting_tasks.py +++ b/tests/app/celery/test_reporting_tasks.py @@ -10,7 +10,7 @@ from app.celery.reporting_tasks import ( create_nightly_billing, create_nightly_billing_for_day, create_nightly_notification_status, - create_nightly_notification_status_for_day, + create_nightly_notification_status_for_service_and_day, ) from app.config import QueueNames from app.dao.fact_billing_dao import get_rate @@ -62,8 +62,8 @@ def test_create_nightly_billing_triggers_tasks_for_days(notify_api, mocker, day_ @freeze_time('2019-08-01') -def test_create_nightly_notification_status_triggers_tasks_for_days(notify_api, mocker): - mock_celery = mocker.patch('app.celery.reporting_tasks.create_nightly_notification_status_for_day') +def test_create_nightly_notification_status_triggers_tasks(notify_api, sample_service, mocker): + mock_celery = mocker.patch('app.celery.reporting_tasks.create_nightly_notification_status_for_service_and_day') create_nightly_notification_status() assert mock_celery.apply_async.call_count == ( @@ -77,13 +77,21 @@ def test_create_nightly_notification_status_triggers_tasks_for_days(notify_api, [SMS_TYPE, EMAIL_TYPE, LETTER_TYPE] ): mock_celery.apply_async.assert_any_call( - kwargs={'process_day': process_date, 'notification_type': notification_type}, + kwargs={ + 'process_day': process_date, + 'notification_type': notification_type, + 'service_id': sample_service.id, + }, queue=QueueNames.REPORTING ) for process_date in ['2019-07-27', '2019-07-26', '2019-07-25', '2019-07-24', '2019-07-23', '2019-07-22']: mock_celery.apply_async.assert_any_call( - kwargs={'process_day': process_date, 'notification_type': LETTER_TYPE}, + kwargs={ + 'process_day': process_date, + 'notification_type': LETTER_TYPE, + 'service_id': sample_service.id, + }, queue=QueueNames.REPORTING ) @@ -501,7 +509,7 @@ def test_create_nightly_billing_for_day_update_when_record_exists( assert records[0].updated_at -def test_create_nightly_notification_status_for_day(notify_db_session): +def test_create_nightly_notification_status_for_service_and_day(notify_db_session): first_service = create_service(service_name='First Service') first_template = create_template(service=first_service) second_service = create_service(service_name='second Service') @@ -532,9 +540,9 @@ def test_create_nightly_notification_status_for_day(notify_db_session): assert len(FactNotificationStatus.query.all()) == 0 - create_nightly_notification_status_for_day(str(process_day), 'sms') - create_nightly_notification_status_for_day(str(process_day), 'email') - create_nightly_notification_status_for_day(str(process_day), 'letter') + create_nightly_notification_status_for_service_and_day(str(process_day), first_service.id, 'sms') + create_nightly_notification_status_for_service_and_day(str(process_day), second_service.id, 'email') + create_nightly_notification_status_for_service_and_day(str(process_day), third_service.id, 'letter') new_fact_data = FactNotificationStatus.query.order_by( FactNotificationStatus.notification_type @@ -569,13 +577,13 @@ def test_create_nightly_notification_status_for_day(notify_db_session): assert new_fact_data[2].key_type == KEY_TYPE_NORMAL -def test_create_nightly_notification_status_for_day_overwrites_old_data(notify_db_session): +def test_create_nightly_notification_status_for_service_and_day_overwrites_old_data(notify_db_session): first_service = create_service(service_name='First Service') first_template = create_template(service=first_service) create_notification(template=first_template, status='delivered') process_day = date.today() - create_nightly_notification_status_for_day(str(process_day), 'sms') + create_nightly_notification_status_for_service_and_day(str(process_day), first_service.id, 'sms') new_fact_data = FactNotificationStatus.query.order_by( FactNotificationStatus.bst_date, @@ -586,7 +594,7 @@ def test_create_nightly_notification_status_for_day_overwrites_old_data(notify_d assert new_fact_data[0].notification_count == 1 create_notification(template=first_template, status='delivered') - create_nightly_notification_status_for_day(str(process_day), 'sms') + create_nightly_notification_status_for_service_and_day(str(process_day), first_service.id, 'sms') updated_fact_data = FactNotificationStatus.query.order_by( FactNotificationStatus.bst_date, @@ -599,7 +607,7 @@ def test_create_nightly_notification_status_for_day_overwrites_old_data(notify_d # the job runs at 12:30am London time. 04/01 is in BST. @freeze_time('2019-04-01T23:30') -def test_create_nightly_notification_status_for_day_respects_bst(sample_template): +def test_create_nightly_notification_status_for_service_and_day_respects_bst(sample_template): create_notification(sample_template, status='delivered', created_at=datetime(2019, 4, 1, 23, 0)) # too new create_notification(sample_template, status='created', created_at=datetime(2019, 4, 1, 22, 59)) @@ -607,7 +615,7 @@ def test_create_nightly_notification_status_for_day_respects_bst(sample_template create_notification(sample_template, status='delivered', created_at=datetime(2019, 3, 31, 22, 59)) # too old - create_nightly_notification_status_for_day('2019-04-01', 'sms') + create_nightly_notification_status_for_service_and_day('2019-04-01', sample_template.service_id, 'sms') noti_status = FactNotificationStatus.query.order_by(FactNotificationStatus.bst_date).all() assert len(noti_status) == 1