darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2025-12-18 06:01:44 -05:00
Code Issues Packages Projects Releases Wiki Activity

Delete scheduled task to populate monthly_billing

Browse Source
This commit is contained in:
Pea Tyczynska
2018-07-23 14:57:44 +01:00
parent 1d2e086109
commit c0f309a2a6
4 changed files with 1 additions and 188 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
app/celery/scheduled_tasks.py
View File

@@ -380,18 +380,6 @@ def raise_alert_if_letter_notifications_still_sending():
current_app.logger.info(message)
@notify_celery.task(name="populate_monthly_billing")
@statsd(namespace="tasks")
def populate_monthly_billing():
# for every service with billable units this month update billing totals for yesterday
# this will overwrite the existing amount.
yesterday = datetime.utcnow() - timedelta(days=1)
yesterday_in_bst = convert_utc_to_bst(yesterday)
start_date, end_date = get_month_start_and_end_date_in_utc(yesterday_in_bst)
services = get_service_ids_that_need_billing_populated(start_date=start_date, end_date=end_date)
[create_or_update_monthly_billing(service_id=s.service_id, billing_month=end_date) for s in services]
@notify_celery.task(name="run-letter-jobs")
@statsd(namespace="tasks")
def run_letter_jobs():

89
app/commands.py
View File

@@ -24,11 +24,7 @@ from app.dao.fact_billing_dao import (
update_fact_billing,
delete_billing_data_for_service_for_day
)
from app.dao.monthly_billing_dao import (
create_or_update_monthly_billing,
get_monthly_billing_by_notification_type,
get_service_ids_that_need_billing_populated
)
from app.dao.provider_rates_dao import create_provider_rates as dao_create_provider_rates
from app.dao.service_callback_api_dao import get_service_delivery_status_callback_api_for_service
from app.dao.services_dao import (
@@ -191,51 +187,6 @@ def fix_notification_statuses_not_in_sync():
result = db.session.execute(subq_hist).fetchall()
@notify_command()
@click.option('-y', '--year', required=True, help="e.g. 2017", type=int)
@click.option('-s', '--service_id', required=False, help="Enter the service id", type=click.UUID)
@click.option('-m', '--month', required=False, help="e.g. 1 for January", type=int)
def populate_monthly_billing(year, service_id=None, month=None):
"""
Populate monthly billing table for all services for a given year.
If service_id and month provided then only rebuild monthly billing for that month.
"""
def populate(service_id, year, month):
create_or_update_monthly_billing(service_id, datetime(year, month, 1))
sms_res = get_monthly_billing_by_notification_type(
service_id, datetime(year, month, 1), SMS_TYPE
)
email_res = get_monthly_billing_by_notification_type(
service_id, datetime(year, month, 1), EMAIL_TYPE
)
letter_res = get_monthly_billing_by_notification_type(
service_id, datetime(year, month, 1), 'letter'
)
print("Finished populating data for {}-{} for service id {}".format(month, year, str(service_id)))
print('SMS: {}'.format(sms_res.monthly_totals))
print('Email: {}'.format(email_res.monthly_totals))
print('Letter: {}'.format(letter_res.monthly_totals))
if service_id and month:
populate(service_id, year, month)
else:
service_ids = get_service_ids_that_need_billing_populated(
start_date=datetime(2016, 5, 1), end_date=datetime(2017, 8, 16)
)
start, end = 1, 13
if year == 2016:
start = 4
for service_id in service_ids:
print('Starting to populate data for service {}'.format(str(service_id)))
print('Starting populating monthly billing for {}'.format(year))
for i in range(start, end):
print('Population for {}-{}'.format(i, year))
populate(service_id, year, i)
@notify_command()
@click.option('-s', '--start_date', required=True, help="start date inclusive", type=click_dt(format='%Y-%m-%d'))
@click.option('-e', '--end_date', required=True, help="end date inclusive", type=click_dt(format='%Y-%m-%d'))
@@ -560,44 +511,6 @@ def rebuild_ft_billing_for_day(service_id, day):
rebuild_ft_data(day, row.service_id)
@notify_command(name='compare-ft-billing-to-monthly-billing')
@click.option('-y', '--year', required=True)
@click.option('-s', '--service_id', required=False, type=click.UUID)
def compare_ft_billing_to_monthly_billing(year, service_id=None):
"""
This command checks the results of monthly_billing to ft_billing for the given year.
If service id is not included all services are compared for the given year.
"""
def compare_monthly_billing_to_ft_billing(ft_billing_resp, monthly_billing_resp):
# Remove the rows with 0 billing_units and rate, ft_billing doesn't populate those rows.
mo_json = json.loads(monthly_billing_resp.get_data(as_text=True))
rm_zero_rows = [x for x in mo_json if x['billing_units'] != 0 and x['rate'] != 0]
try:
assert rm_zero_rows == json.loads(ft_billing_resp.get_data(as_text=True))
except AssertionError:
print("Comparison failed for service: {} and year: {}".format(service_id, year))
if not service_id:
start_date, end_date = get_financial_year(year=int(year))
services = get_service_ids_that_need_billing_populated(start_date, end_date)
for service_id in services:
with current_app.test_request_context(
path='/service/{}/billing/monthly-usage?year={}'.format(service_id, year)):
monthly_billing_response = get_yearly_usage_by_month(service_id)
with current_app.test_request_context(
path='/service/{}/billing/ft-monthly-usage?year={}'.format(service_id, year)):
ft_billing_response = get_yearly_usage_by_monthly_from_ft_billing(service_id)
compare_monthly_billing_to_ft_billing(ft_billing_response, monthly_billing_response)
else:
with current_app.test_request_context(
path='/service/{}/billing/monthly-usage?year={}'.format(service_id, year)):
monthly_billing_response = get_yearly_usage_by_month(service_id)
with current_app.test_request_context(
path='/service/{}/billing/ft-monthly-usage?year={}'.format(service_id, year)):
ft_billing_response = get_yearly_usage_by_monthly_from_ft_billing(service_id)
compare_monthly_billing_to_ft_billing(ft_billing_response, monthly_billing_response)
@notify_command(name='migrate-data-to-ft-notification-status')
@click.option('-s', '--start_date', required=True, help="start date inclusive", type=click_dt(format='%Y-%m-%d'))
@click.option('-e', '--end_date', required=True, help="end date inclusive", type=click_dt(format='%Y-%m-%d'))

5
app/config.py
View File

@@ -238,11 +238,6 @@ class Config(object):
'schedule': crontab(hour=4, minute=40),
'options': {'queue': QueueNames.PERIODIC}
},
'populate_monthly_billing': {
'task': 'populate_monthly_billing',
'schedule': crontab(hour=5, minute=10),
'options': {'queue': QueueNames.PERIODIC}
},
'raise-alert-if-letter-notifications-still-sending': {
'task': 'raise-alert-if-letter-notifications-still-sending',
'schedule': crontab(hour=16, minute=30),

83
tests/app/celery/test_scheduled_tasks.py
View File

@@ -27,7 +27,6 @@ from app.celery.scheduled_tasks import (
run_scheduled_jobs,
run_letter_jobs,
trigger_letter_pdfs_for_day,
populate_monthly_billing,
s3,
send_daily_performance_platform_stats,
send_scheduled_notifications,
@@ -48,7 +47,6 @@ from app.dao.provider_details_dao import (
)
from app.exceptions import NotificationTechnicalFailureException
from app.models import (
MonthlyBilling,
NotificationHistory,
Service,
StatsTemplateUsageByMonth,
@@ -125,8 +123,6 @@ def test_should_have_decorated_tasks_functions():
'remove_transformed_dvla_files'
assert delete_dvla_response_files_older_than_seven_days.__wrapped__.__name__ == \
'delete_dvla_response_files_older_than_seven_days'
assert populate_monthly_billing.__wrapped__.__name__ == \
'populate_monthly_billing'
@pytest.fixture(scope='function')
@@ -751,85 +747,6 @@ def test_tuesday_alert_if_letter_notifications_still_sending_reports_friday_lett
)
@freeze_time("2017-07-12 02:00:00")
def test_populate_monthly_billing_populates_correctly(sample_template):
yesterday = datetime(2017, 7, 11, 13, 30)
jul_month_start = datetime(2017, 6, 30, 23)
jul_month_end = datetime(2017, 7, 31, 22, 59, 59, 99999)
create_rate(datetime(2016, 1, 1), 0.0123, 'sms')
create_notification(template=sample_template, status='delivered', created_at=yesterday)
create_notification(template=sample_template, status='delivered', created_at=yesterday - timedelta(days=1))
create_notification(template=sample_template, status='delivered', created_at=yesterday + timedelta(days=1))
# not included in billing
create_notification(template=sample_template, status='delivered', created_at=yesterday - timedelta(days=30))
populate_monthly_billing()
monthly_billing = MonthlyBilling.query.order_by(MonthlyBilling.notification_type).all()
assert len(monthly_billing) == 3
assert monthly_billing[0].service_id == sample_template.service_id
assert monthly_billing[0].start_date == jul_month_start
assert monthly_billing[0].end_date == jul_month_end
assert monthly_billing[0].notification_type == 'email'
assert monthly_billing[0].monthly_totals == []
assert monthly_billing[1].service_id == sample_template.service_id
assert monthly_billing[1].start_date == jul_month_start
assert monthly_billing[1].end_date == jul_month_end
assert monthly_billing[1].notification_type == 'sms'
assert sorted(monthly_billing[1].monthly_totals[0]) == sorted(
{
'international': False,
'rate_multiplier': 1,
'billing_units': 3,
'rate': 0.0123,
'total_cost': 0.0369
}
)
assert monthly_billing[2].service_id == sample_template.service_id
assert monthly_billing[2].start_date == jul_month_start
assert monthly_billing[2].end_date == jul_month_end
assert monthly_billing[2].notification_type == 'letter'
assert monthly_billing[2].monthly_totals == []
@freeze_time("2016-04-01 23:00:00")
def test_populate_monthly_billing_updates_correct_month_in_bst(sample_template):
yesterday = datetime.utcnow() - timedelta(days=1)
apr_month_start = datetime(2016, 3, 31, 23)
apr_month_end = datetime(2016, 4, 30, 22, 59, 59, 99999)
create_rate(datetime(2016, 1, 1), 0.0123, 'sms')
create_notification(template=sample_template, status='delivered', created_at=yesterday)
populate_monthly_billing()
monthly_billing = MonthlyBilling.query.order_by(MonthlyBilling.notification_type).all()
assert len(monthly_billing) == 3
assert monthly_billing[0].service_id == sample_template.service_id
assert monthly_billing[0].start_date == apr_month_start
assert monthly_billing[0].end_date == apr_month_end
assert monthly_billing[0].notification_type == 'email'
assert monthly_billing[0].monthly_totals == []
assert monthly_billing[1].service_id == sample_template.service_id
assert monthly_billing[1].start_date == apr_month_start
assert monthly_billing[1].end_date == apr_month_end
assert monthly_billing[1].notification_type == 'sms'
assert monthly_billing[1].monthly_totals[0]['billing_units'] == 1
assert monthly_billing[1].monthly_totals[0]['total_cost'] == 0.0123
assert monthly_billing[2].service_id == sample_template.service_id
assert monthly_billing[2].start_date == apr_month_start
assert monthly_billing[2].end_date == apr_month_end
assert monthly_billing[2].notification_type == 'letter'
assert monthly_billing[2].monthly_totals == []
def test_run_letter_jobs(client, mocker, sample_letter_template):
jobs = [create_job(template=sample_letter_template, job_status=JOB_STATUS_READY_TO_SEND),
create_job(template=sample_letter_template, job_status=JOB_STATUS_READY_TO_SEND)]