From c0f309a2a6b65586fd00e1fc12d1be3cce57dd9c Mon Sep 17 00:00:00 2001 From: Pea Tyczynska Date: Mon, 23 Jul 2018 14:57:44 +0100 Subject: [PATCH] Delete scheduled task to populate monthly_billing --- app/celery/scheduled_tasks.py | 12 ---- app/commands.py | 89 +----------------------- app/config.py | 5 -- tests/app/celery/test_scheduled_tasks.py | 83 ---------------------- 4 files changed, 1 insertion(+), 188 deletions(-) diff --git a/app/celery/scheduled_tasks.py b/app/celery/scheduled_tasks.py index 5de26e70e..85276aa30 100644 --- a/app/celery/scheduled_tasks.py +++ b/app/celery/scheduled_tasks.py @@ -380,18 +380,6 @@ def raise_alert_if_letter_notifications_still_sending(): current_app.logger.info(message) -@notify_celery.task(name="populate_monthly_billing") -@statsd(namespace="tasks") -def populate_monthly_billing(): - # for every service with billable units this month update billing totals for yesterday - # this will overwrite the existing amount. - yesterday = datetime.utcnow() - timedelta(days=1) - yesterday_in_bst = convert_utc_to_bst(yesterday) - start_date, end_date = get_month_start_and_end_date_in_utc(yesterday_in_bst) - services = get_service_ids_that_need_billing_populated(start_date=start_date, end_date=end_date) - [create_or_update_monthly_billing(service_id=s.service_id, billing_month=end_date) for s in services] - - @notify_celery.task(name="run-letter-jobs") @statsd(namespace="tasks") def run_letter_jobs(): diff --git a/app/commands.py b/app/commands.py index 982fbdc97..8b1a9d286 100644 --- a/app/commands.py +++ b/app/commands.py @@ -24,11 +24,7 @@ from app.dao.fact_billing_dao import ( update_fact_billing, delete_billing_data_for_service_for_day ) -from app.dao.monthly_billing_dao import ( - create_or_update_monthly_billing, - get_monthly_billing_by_notification_type, - get_service_ids_that_need_billing_populated -) + from app.dao.provider_rates_dao import create_provider_rates as dao_create_provider_rates from app.dao.service_callback_api_dao import get_service_delivery_status_callback_api_for_service from app.dao.services_dao import ( @@ -191,51 +187,6 @@ def fix_notification_statuses_not_in_sync(): result = db.session.execute(subq_hist).fetchall() -@notify_command() -@click.option('-y', '--year', required=True, help="e.g. 2017", type=int) -@click.option('-s', '--service_id', required=False, help="Enter the service id", type=click.UUID) -@click.option('-m', '--month', required=False, help="e.g. 1 for January", type=int) -def populate_monthly_billing(year, service_id=None, month=None): - """ - Populate monthly billing table for all services for a given year. - If service_id and month provided then only rebuild monthly billing for that month. - """ - def populate(service_id, year, month): - create_or_update_monthly_billing(service_id, datetime(year, month, 1)) - sms_res = get_monthly_billing_by_notification_type( - service_id, datetime(year, month, 1), SMS_TYPE - ) - email_res = get_monthly_billing_by_notification_type( - service_id, datetime(year, month, 1), EMAIL_TYPE - ) - letter_res = get_monthly_billing_by_notification_type( - service_id, datetime(year, month, 1), 'letter' - ) - - print("Finished populating data for {}-{} for service id {}".format(month, year, str(service_id))) - print('SMS: {}'.format(sms_res.monthly_totals)) - print('Email: {}'.format(email_res.monthly_totals)) - print('Letter: {}'.format(letter_res.monthly_totals)) - - if service_id and month: - populate(service_id, year, month) - else: - service_ids = get_service_ids_that_need_billing_populated( - start_date=datetime(2016, 5, 1), end_date=datetime(2017, 8, 16) - ) - start, end = 1, 13 - - if year == 2016: - start = 4 - - for service_id in service_ids: - print('Starting to populate data for service {}'.format(str(service_id))) - print('Starting populating monthly billing for {}'.format(year)) - for i in range(start, end): - print('Population for {}-{}'.format(i, year)) - populate(service_id, year, i) - - @notify_command() @click.option('-s', '--start_date', required=True, help="start date inclusive", type=click_dt(format='%Y-%m-%d')) @click.option('-e', '--end_date', required=True, help="end date inclusive", type=click_dt(format='%Y-%m-%d')) @@ -560,44 +511,6 @@ def rebuild_ft_billing_for_day(service_id, day): rebuild_ft_data(day, row.service_id) -@notify_command(name='compare-ft-billing-to-monthly-billing') -@click.option('-y', '--year', required=True) -@click.option('-s', '--service_id', required=False, type=click.UUID) -def compare_ft_billing_to_monthly_billing(year, service_id=None): - """ - This command checks the results of monthly_billing to ft_billing for the given year. - If service id is not included all services are compared for the given year. - """ - def compare_monthly_billing_to_ft_billing(ft_billing_resp, monthly_billing_resp): - # Remove the rows with 0 billing_units and rate, ft_billing doesn't populate those rows. - mo_json = json.loads(monthly_billing_resp.get_data(as_text=True)) - rm_zero_rows = [x for x in mo_json if x['billing_units'] != 0 and x['rate'] != 0] - try: - assert rm_zero_rows == json.loads(ft_billing_resp.get_data(as_text=True)) - except AssertionError: - print("Comparison failed for service: {} and year: {}".format(service_id, year)) - - if not service_id: - start_date, end_date = get_financial_year(year=int(year)) - services = get_service_ids_that_need_billing_populated(start_date, end_date) - for service_id in services: - with current_app.test_request_context( - path='/service/{}/billing/monthly-usage?year={}'.format(service_id, year)): - monthly_billing_response = get_yearly_usage_by_month(service_id) - with current_app.test_request_context( - path='/service/{}/billing/ft-monthly-usage?year={}'.format(service_id, year)): - ft_billing_response = get_yearly_usage_by_monthly_from_ft_billing(service_id) - compare_monthly_billing_to_ft_billing(ft_billing_response, monthly_billing_response) - else: - with current_app.test_request_context( - path='/service/{}/billing/monthly-usage?year={}'.format(service_id, year)): - monthly_billing_response = get_yearly_usage_by_month(service_id) - with current_app.test_request_context( - path='/service/{}/billing/ft-monthly-usage?year={}'.format(service_id, year)): - ft_billing_response = get_yearly_usage_by_monthly_from_ft_billing(service_id) - compare_monthly_billing_to_ft_billing(ft_billing_response, monthly_billing_response) - - @notify_command(name='migrate-data-to-ft-notification-status') @click.option('-s', '--start_date', required=True, help="start date inclusive", type=click_dt(format='%Y-%m-%d')) @click.option('-e', '--end_date', required=True, help="end date inclusive", type=click_dt(format='%Y-%m-%d')) diff --git a/app/config.py b/app/config.py index 0a490588f..f4486121e 100644 --- a/app/config.py +++ b/app/config.py @@ -238,11 +238,6 @@ class Config(object): 'schedule': crontab(hour=4, minute=40), 'options': {'queue': QueueNames.PERIODIC} }, - 'populate_monthly_billing': { - 'task': 'populate_monthly_billing', - 'schedule': crontab(hour=5, minute=10), - 'options': {'queue': QueueNames.PERIODIC} - }, 'raise-alert-if-letter-notifications-still-sending': { 'task': 'raise-alert-if-letter-notifications-still-sending', 'schedule': crontab(hour=16, minute=30), diff --git a/tests/app/celery/test_scheduled_tasks.py b/tests/app/celery/test_scheduled_tasks.py index 0b192972d..30b3308e8 100644 --- a/tests/app/celery/test_scheduled_tasks.py +++ b/tests/app/celery/test_scheduled_tasks.py @@ -27,7 +27,6 @@ from app.celery.scheduled_tasks import ( run_scheduled_jobs, run_letter_jobs, trigger_letter_pdfs_for_day, - populate_monthly_billing, s3, send_daily_performance_platform_stats, send_scheduled_notifications, @@ -48,7 +47,6 @@ from app.dao.provider_details_dao import ( ) from app.exceptions import NotificationTechnicalFailureException from app.models import ( - MonthlyBilling, NotificationHistory, Service, StatsTemplateUsageByMonth, @@ -125,8 +123,6 @@ def test_should_have_decorated_tasks_functions(): 'remove_transformed_dvla_files' assert delete_dvla_response_files_older_than_seven_days.__wrapped__.__name__ == \ 'delete_dvla_response_files_older_than_seven_days' - assert populate_monthly_billing.__wrapped__.__name__ == \ - 'populate_monthly_billing' @pytest.fixture(scope='function') @@ -751,85 +747,6 @@ def test_tuesday_alert_if_letter_notifications_still_sending_reports_friday_lett ) -@freeze_time("2017-07-12 02:00:00") -def test_populate_monthly_billing_populates_correctly(sample_template): - yesterday = datetime(2017, 7, 11, 13, 30) - jul_month_start = datetime(2017, 6, 30, 23) - jul_month_end = datetime(2017, 7, 31, 22, 59, 59, 99999) - create_rate(datetime(2016, 1, 1), 0.0123, 'sms') - - create_notification(template=sample_template, status='delivered', created_at=yesterday) - create_notification(template=sample_template, status='delivered', created_at=yesterday - timedelta(days=1)) - create_notification(template=sample_template, status='delivered', created_at=yesterday + timedelta(days=1)) - # not included in billing - create_notification(template=sample_template, status='delivered', created_at=yesterday - timedelta(days=30)) - - populate_monthly_billing() - - monthly_billing = MonthlyBilling.query.order_by(MonthlyBilling.notification_type).all() - - assert len(monthly_billing) == 3 - - assert monthly_billing[0].service_id == sample_template.service_id - assert monthly_billing[0].start_date == jul_month_start - assert monthly_billing[0].end_date == jul_month_end - assert monthly_billing[0].notification_type == 'email' - assert monthly_billing[0].monthly_totals == [] - - assert monthly_billing[1].service_id == sample_template.service_id - assert monthly_billing[1].start_date == jul_month_start - assert monthly_billing[1].end_date == jul_month_end - assert monthly_billing[1].notification_type == 'sms' - assert sorted(monthly_billing[1].monthly_totals[0]) == sorted( - { - 'international': False, - 'rate_multiplier': 1, - 'billing_units': 3, - 'rate': 0.0123, - 'total_cost': 0.0369 - } - ) - - assert monthly_billing[2].service_id == sample_template.service_id - assert monthly_billing[2].start_date == jul_month_start - assert monthly_billing[2].end_date == jul_month_end - assert monthly_billing[2].notification_type == 'letter' - assert monthly_billing[2].monthly_totals == [] - - -@freeze_time("2016-04-01 23:00:00") -def test_populate_monthly_billing_updates_correct_month_in_bst(sample_template): - yesterday = datetime.utcnow() - timedelta(days=1) - apr_month_start = datetime(2016, 3, 31, 23) - apr_month_end = datetime(2016, 4, 30, 22, 59, 59, 99999) - create_rate(datetime(2016, 1, 1), 0.0123, 'sms') - create_notification(template=sample_template, status='delivered', created_at=yesterday) - populate_monthly_billing() - - monthly_billing = MonthlyBilling.query.order_by(MonthlyBilling.notification_type).all() - - assert len(monthly_billing) == 3 - - assert monthly_billing[0].service_id == sample_template.service_id - assert monthly_billing[0].start_date == apr_month_start - assert monthly_billing[0].end_date == apr_month_end - assert monthly_billing[0].notification_type == 'email' - assert monthly_billing[0].monthly_totals == [] - - assert monthly_billing[1].service_id == sample_template.service_id - assert monthly_billing[1].start_date == apr_month_start - assert monthly_billing[1].end_date == apr_month_end - assert monthly_billing[1].notification_type == 'sms' - assert monthly_billing[1].monthly_totals[0]['billing_units'] == 1 - assert monthly_billing[1].monthly_totals[0]['total_cost'] == 0.0123 - - assert monthly_billing[2].service_id == sample_template.service_id - assert monthly_billing[2].start_date == apr_month_start - assert monthly_billing[2].end_date == apr_month_end - assert monthly_billing[2].notification_type == 'letter' - assert monthly_billing[2].monthly_totals == [] - - def test_run_letter_jobs(client, mocker, sample_letter_template): jobs = [create_job(template=sample_letter_template, job_status=JOB_STATUS_READY_TO_SEND), create_job(template=sample_letter_template, job_status=JOB_STATUS_READY_TO_SEND)]